--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:50:40.493830Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-a7932c-predictor-648777d896-sq28d namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: c9b64a1d-9f71-4364-9f9e-e76d7789ecf2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-a7932c-predictor-648777d896-sq28d to ip-10-0-134-249.ec2.internal metadata: creationTimestamp: "2026-04-24T21:50:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:50:40Z" name: isvc-init-fail-a7932c-predictor-648777d896-sq28d.18a9696bfc8c3c3b namespace: kserve-ci-e2e-test resourceVersion: "19653" uid: 39dd822f-f43e-40db-afcc-a3db476289c1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6688756845-wvlxd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-a7932c-predictor-648777d896-sq28d namespace: kserve-ci-e2e-test resourceVersion: "19656" uid: c9b64a1d-9f71-4364-9f9e-e76d7789ecf2 kind: Event lastTimestamp: "2026-04-24T21:50:40Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:50:40Z" name: isvc-init-fail-a7932c-predictor-648777d896-sq28d.18a9696c16097bae namespace: kserve-ci-e2e-test resourceVersion: "19665" uid: 5ea9671a-ccc3-41fa-9994-d0341b4620aa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:50:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-a7932c-predictor-648777d896-sq28d namespace: kserve-ci-e2e-test resourceVersion: "19650" uid: c9b64a1d-9f71-4364-9f9e-e76d7789ecf2 kind: Event lastTimestamp: "2026-04-24T21:50:46Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0dd2798f9cdbffc0563bd148612201df0e589ff8c26a6b19d321fd120fc5c097" already present on machine metadata: creationTimestamp: "2026-04-24T21:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:46Z" name: isvc-init-fail-a7932c-predictor-648777d896-sq28d.18a9696c179dd38f namespace: kserve-ci-e2e-test resourceVersion: "19717" uid: f4f5163f-8eca-4245-a916-494ffb77a1cc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:50:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-a7932c-predictor-648777d896-sq28d namespace: kserve-ci-e2e-test resourceVersion: "19650" uid: c9b64a1d-9f71-4364-9f9e-e76d7789ecf2 kind: Event lastTimestamp: "2026-04-24T21:50:46Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:46Z" name: isvc-init-fail-a7932c-predictor-648777d896-sq28d.18a9696c1a457199 namespace: kserve-ci-e2e-test resourceVersion: "19720" uid: 6b020759-da94-4b0c-a6d5-1445c71c1e7f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:50:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-a7932c-predictor-648777d896-sq28d namespace: kserve-ci-e2e-test resourceVersion: "19650" uid: c9b64a1d-9f71-4364-9f9e-e76d7789ecf2 kind: Event lastTimestamp: "2026-04-24T21:50:46Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:46Z" name: isvc-init-fail-a7932c-predictor-648777d896-sq28d.18a9696c1ab4a18b namespace: kserve-ci-e2e-test resourceVersion: "19723" uid: ba930f4b-9b16-4b65-9a43-23f588292980 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-a7932c-predictor-648777d896-sq28d namespace: kserve-ci-e2e-test resourceVersion: "19650" uid: c9b64a1d-9f71-4364-9f9e-e76d7789ecf2 kind: Event lastTimestamp: "2026-04-24T21:50:49Z" message: Back-off restarting failed container storage-initializer in pod isvc-init-fail-a7932c-predictor-648777d896-sq28d_kserve-ci-e2e-test(c9b64a1d-9f71-4364-9f9e-e76d7789ecf2) metadata: creationTimestamp: "2026-04-24T21:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:49Z" name: isvc-init-fail-a7932c-predictor-648777d896-sq28d.18a9696e0e19c6b7 namespace: kserve-ci-e2e-test resourceVersion: "19745" uid: 122c2dcb-6d07-425f-9ddf-79c29473ec5c reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-a7932c-predictor-648777d896 namespace: kserve-ci-e2e-test resourceVersion: "19636" uid: 26044540-c4f2-42ca-b666-d46812c81a1d kind: Event lastTimestamp: "2026-04-24T21:50:40Z" message: 'Created pod: isvc-init-fail-a7932c-predictor-648777d896-sq28d' metadata: creationTimestamp: "2026-04-24T21:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:40Z" name: isvc-init-fail-a7932c-predictor-648777d896.18a9696bfc465262 namespace: kserve-ci-e2e-test resourceVersion: "19651" uid: bf2e4ab9-0b10-4a42-b3d6-ac1df6292807 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-a7932c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19630" uid: 1193a4e6-37ac-477a-8864-25bab491855d kind: Event lastTimestamp: "2026-04-24T21:50:40Z" message: Scaled up replica set isvc-init-fail-a7932c-predictor-648777d896 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:40Z" name: isvc-init-fail-a7932c-predictor.18a9696bf8eec03d namespace: kserve-ci-e2e-test resourceVersion: "19638" uid: 1ec2d450-3445-4f4f-9545-3852e9428910 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T21:50:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-a7932c namespace: kserve-ci-e2e-test resourceVersion: "19598" uid: 4f01f2ea-e12b-42c4-acae-f3c2145e3467 kind: Event lastTimestamp: "2026-04-24T21:50:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:50Z" name: isvc-init-fail-a7932c.18a9696bf8c46b40 namespace: kserve-ci-e2e-test resourceVersion: "19754" uid: 2b36c763-7007-4c4d-a526-ec4a0bff29f7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-a7932c namespace: kserve-ci-e2e-test resourceVersion: "19598" uid: 4f01f2ea-e12b-42c4-acae-f3c2145e3467 kind: Event lastTimestamp: "2026-04-24T21:50:40Z" message: 'Failed to update status for InferenceService "isvc-init-fail-a7932c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-a7932c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:40Z" name: isvc-init-fail-a7932c.18a9696bf919776f namespace: kserve-ci-e2e-test resourceVersion: "19641" uid: 0e626321-48b2-43dd-b0da-609d403ca5f8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-a7932c namespace: kserve-ci-e2e-test resourceVersion: "19598" uid: 4f01f2ea-e12b-42c4-acae-f3c2145e3467 kind: Event lastTimestamp: "2026-04-24T21:50:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-a7932c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:40Z" name: isvc-init-fail-a7932c.18a9696bf919bc02 namespace: kserve-ci-e2e-test resourceVersion: "19642" uid: 9cbf2694-367d-4a69-b1f6-7870e619a5d0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:45:16.689505Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16943" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 to ip-10-0-134-249.ec2.internal metadata: creationTimestamp: "2026-04-24T21:45:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:45:16Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a96920984e5ce7 namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 245f7b97-635b-454e-80a0-eaed1b42826c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6688756845-wvlxd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: "2026-04-24T21:45:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-raw-fb0e6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:16Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a96920a46ea0e3 namespace: kserve-ci-e2e-test resourceVersion: "16954" uid: a48de56f-7bde-4781-968c-c1c7830cfd9e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:17Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16951" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: "2026-04-24T21:45:17Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:45:17Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a96920d5179df0 namespace: kserve-ci-e2e-test resourceVersion: "16967" uid: 566cbf08-8007-44a1-8e32-ecb1d85a5809 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: "2026-04-24T21:45:17Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0dd2798f9cdbffc0563bd148612201df0e589ff8c26a6b19d321fd120fc5c097" already present on machine metadata: creationTimestamp: "2026-04-24T21:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:17Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a96920d69627c6 namespace: kserve-ci-e2e-test resourceVersion: "16969" uid: 29e0b4a0-e640-4c58-b56e-7c754d7b0516 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: "2026-04-24T21:45:17Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:17Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a96920d971eae5 namespace: kserve-ci-e2e-test resourceVersion: "16970" uid: 5662768d-cb78-4da9-b141-4ae2480ba686 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: "2026-04-24T21:45:17Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:17Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a96920d9e3a0f3 namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: 8f57520b-b9c0-4f6d-9e97-42a521671aed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: "2026-04-24T21:45:22Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:22Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a96921e1fce34d namespace: kserve-ci-e2e-test resourceVersion: "17013" uid: c39c5019-872a-4dec-aa08-b14e3665f0ad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: "2026-04-24T21:45:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:22Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a96921e50a4277 namespace: kserve-ci-e2e-test resourceVersion: "17015" uid: 392bb880-40bd-439e-893d-573a2cfdbaab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: "2026-04-24T21:45:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:22Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a96921e57b1cf0 namespace: kserve-ci-e2e-test resourceVersion: "17018" uid: e540b02f-5801-43eb-b2d6-3577d0a59721 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: "2026-04-24T21:45:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:22Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a96921e5878270 namespace: kserve-ci-e2e-test resourceVersion: "17019" uid: 08ef7c62-46cb-44ed-bf7b-65b56367d7d2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: "2026-04-24T21:45:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:22Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a96921e8e82399 namespace: kserve-ci-e2e-test resourceVersion: "17022" uid: 14696ce9-b2af-4151-9fcc-5c30400b5105 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: "2026-04-24T21:45:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:22Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a96921e9584e52 namespace: kserve-ci-e2e-test resourceVersion: "17024" uid: 6783c5f6-9cfb-49d1-93e0-0cf5e029bf28 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: "2026-04-24T21:45:22Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:f72c005cb705ef76fbeae81cf78e50cb05f1674ac01bdd8893cf8fb48213f3d0" already present on machine metadata: creationTimestamp: "2026-04-24T21:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:22Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a96921e964ca80 namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 2b40a532-e37d-4181-9bfb-61c81a60f70a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: "2026-04-24T21:45:22Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-24T21:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:22Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a96921ec6239d3 namespace: kserve-ci-e2e-test resourceVersion: "17026" uid: e6c13903-e838-4a49-b9cd-775bc16dfb2c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: "2026-04-24T21:45:22Z" message: Started container agent metadata: creationTimestamp: "2026-04-24T21:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:22Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a96921eccd716d namespace: kserve-ci-e2e-test resourceVersion: "17027" uid: 7a0b1461-7f89-4dea-9404-3dbed39d9ad8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T21:45:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: "2026-04-24T21:46:59Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:45:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:59Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a969221df510bb namespace: kserve-ci-e2e-test resourceVersion: "17854" uid: fd93fb2c-b83f-4abb-9e7e-b6e38e256b89 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T21:45:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: "2026-04-24T21:46:59Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:45:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:59Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a969221dff1bc6 namespace: kserve-ci-e2e-test resourceVersion: "17855" uid: da9f9703-3d32-49a7-84ad-af9931659c57 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: "2026-04-24T21:46:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:41Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a969346e7335d5 namespace: kserve-ci-e2e-test resourceVersion: "17664" uid: 3f7f98ec-3623-491e-84bf-283c21df48c6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: "2026-04-24T21:46:41Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-24T21:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:41Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a969346e738f00 namespace: kserve-ci-e2e-test resourceVersion: "17665" uid: a2de3a69-0adb-4b24-8e00-fb99b176389d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: "2026-04-24T21:46:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:41Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a969346e73faff namespace: kserve-ci-e2e-test resourceVersion: "17667" uid: 489c8021-cb2c-46b5-8af5-4ca950ab3b09 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:46:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 7dd3c20b-5f46-4349-ba0f-91f74891d4ba kind: Event lastTimestamp: "2026-04-24T21:46:59Z" message: 'Readiness probe failed: Get "https://10.132.0.40:8643/healthz": dial tcp 10.132.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:46:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:59Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9.18a96934fa0c96e8 namespace: kserve-ci-e2e-test resourceVersion: "17853" uid: 3f6ab0ee-cf9f-4d3c-b778-e4154893ca40 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-fb0e6-predictor-77dc655476 namespace: kserve-ci-e2e-test resourceVersion: "16932" uid: 654e0f5c-908d-42ae-887c-ff6226bab168 kind: Event lastTimestamp: "2026-04-24T21:45:16Z" message: 'Created pod: isvc-logger-raw-fb0e6-predictor-77dc655476-ltfd9' metadata: creationTimestamp: "2026-04-24T21:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:16Z" name: isvc-logger-raw-fb0e6-predictor-77dc655476.18a96920972ba689 namespace: kserve-ci-e2e-test resourceVersion: "16944" uid: 8573f6da-ba6c-4636-bdcf-6ebb23530e50 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-fb0e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "16930" uid: c29b9ea4-ac38-445d-bc9e-f3ad8907fd98 kind: Event lastTimestamp: "2026-04-24T21:45:16Z" message: Scaled up replica set isvc-logger-raw-fb0e6-predictor-77dc655476 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:16Z" name: isvc-logger-raw-fb0e6-predictor.18a9692093702155 namespace: kserve-ci-e2e-test resourceVersion: "16934" uid: 8da8e4ed-bf55-4534-85a9-c62a40a16e12 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:45:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-fb0e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "16931" uid: 5606efa6-b46b-416b-b4f3-df70035eb49b kind: Event lastTimestamp: "2026-04-24T21:46:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:01Z" name: isvc-logger-raw-fb0e6-predictor.18a9692411ea5122 namespace: kserve-ci-e2e-test resourceVersion: "17328" uid: 0a39a46c-11dc-45fc-8cd7-fe489e6032fd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:45:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-fb0e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "16931" uid: 5606efa6-b46b-416b-b4f3-df70035eb49b kind: Event lastTimestamp: "2026-04-24T21:46:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:01Z" name: isvc-logger-raw-fb0e6-predictor.18a96924125b4448 namespace: kserve-ci-e2e-test resourceVersion: "17329" uid: 72ae1d87-e715-4033-8dc3-adc28ff053c1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:46:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-fb0e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "17105" uid: 5606efa6-b46b-416b-b4f3-df70035eb49b kind: Event lastTimestamp: "2026-04-24T21:46:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:31Z" name: isvc-logger-raw-fb0e6-predictor.18a9692e8fc5cf3d namespace: kserve-ci-e2e-test resourceVersion: "17548" uid: 46fc9f83-472a-4c60-b8bc-5e4c76599292 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:46:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-fb0e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "17105" uid: 5606efa6-b46b-416b-b4f3-df70035eb49b kind: Event lastTimestamp: "2026-04-24T21:46:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:31Z" name: isvc-logger-raw-fb0e6-predictor.18a9692e903d0a2b namespace: kserve-ci-e2e-test resourceVersion: "17549" uid: eb095cf1-8b13-4b59-a44f-d3c7c4789bbf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T21:45:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-fb0e6 namespace: kserve-ci-e2e-test resourceVersion: "16920" uid: adc300c8-b32d-4918-a8e3-998de1330bdf kind: Event lastTimestamp: "2026-04-24T21:46:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:46:29Z" name: isvc-logger-raw-fb0e6.18a969209333a247 namespace: kserve-ci-e2e-test resourceVersion: "17529" uid: f2f1c6c7-9e7e-4e18-b5cb-e838fee33eb8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-fb0e6 namespace: kserve-ci-e2e-test resourceVersion: "16920" uid: adc300c8-b32d-4918-a8e3-998de1330bdf kind: Event lastTimestamp: "2026-04-24T21:45:16Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-fb0e6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-fb0e6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:16Z" name: isvc-logger-raw-fb0e6.18a9692093ac9e7f namespace: kserve-ci-e2e-test resourceVersion: "16936" uid: d7b4d271-57a3-4af2-85d5-d1712c7d33ff reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-fb0e6 namespace: kserve-ci-e2e-test resourceVersion: "17526" uid: adc300c8-b32d-4918-a8e3-998de1330bdf kind: Event lastTimestamp: "2026-04-24T21:46:29Z" message: InferenceService [isvc-logger-raw-fb0e6] is Ready metadata: creationTimestamp: "2026-04-24T21:46:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:46:29Z" name: isvc-logger-raw-fb0e6.18a96931827a6410 namespace: kserve-ci-e2e-test resourceVersion: "17528" uid: 3f0a850c-d5a1-40d5-b556-cd184ec4cc38 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:49:02.213122Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74 namespace: kserve-ci-e2e-test resourceVersion: "18774" uid: 7425b4fb-1133-4845-89d5-240d7bb4dc45 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-cfc742-predictor-6f9c45779c-m9v74 to ip-10-0-134-249.ec2.internal metadata: creationTimestamp: "2026-04-24T21:49:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:49:02Z" name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74.18a969551a8fa1a7 namespace: kserve-ci-e2e-test resourceVersion: "18780" uid: 5248a39b-71d0-4478-b2d5-c7ccfffaaa23 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6688756845-wvlxd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74 namespace: kserve-ci-e2e-test resourceVersion: "18782" uid: 7425b4fb-1133-4845-89d5-240d7bb4dc45 kind: Event lastTimestamp: "2026-04-24T21:49:02Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:49:02Z" name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74.18a96955404ec719 namespace: kserve-ci-e2e-test resourceVersion: "18796" uid: 4177df79-a14e-439c-b603-4f6105672a83 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74 namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: 7425b4fb-1133-4845-89d5-240d7bb4dc45 kind: Event lastTimestamp: "2026-04-24T21:49:02Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0dd2798f9cdbffc0563bd148612201df0e589ff8c26a6b19d321fd120fc5c097" already present on machine metadata: creationTimestamp: "2026-04-24T21:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:02Z" name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74.18a9695541b82bcc namespace: kserve-ci-e2e-test resourceVersion: "18798" uid: 7fe977bb-673e-495c-a52d-b16f3ab5b92f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74 namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: 7425b4fb-1133-4845-89d5-240d7bb4dc45 kind: Event lastTimestamp: "2026-04-24T21:49:02Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:02Z" name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74.18a9695545284629 namespace: kserve-ci-e2e-test resourceVersion: "18799" uid: 5b851e79-2197-4cad-915a-ded24e338c8e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74 namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: 7425b4fb-1133-4845-89d5-240d7bb4dc45 kind: Event lastTimestamp: "2026-04-24T21:49:02Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:02Z" name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74.18a9695545adfbcb namespace: kserve-ci-e2e-test resourceVersion: "18800" uid: d8411b50-9e20-4323-bc23-76e571c1249d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74 namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: 7425b4fb-1133-4845-89d5-240d7bb4dc45 kind: Event lastTimestamp: "2026-04-24T21:49:06Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:49:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:06Z" name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74.18a96956371a274a namespace: kserve-ci-e2e-test resourceVersion: "18836" uid: df6990b1-7c7b-499d-99e8-1c5c369e7aae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74 namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: 7425b4fb-1133-4845-89d5-240d7bb4dc45 kind: Event lastTimestamp: "2026-04-24T21:49:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:49:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:07Z" name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74.18a969563a82ab8f namespace: kserve-ci-e2e-test resourceVersion: "18840" uid: 9337469e-60c4-46de-b68f-694510d87de0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74 namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: 7425b4fb-1133-4845-89d5-240d7bb4dc45 kind: Event lastTimestamp: "2026-04-24T21:49:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:49:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:07Z" name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74.18a969563b0a1ccd namespace: kserve-ci-e2e-test resourceVersion: "18841" uid: e4e510c6-2eed-41d4-b0b5-12da06143c29 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74 namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: 7425b4fb-1133-4845-89d5-240d7bb4dc45 kind: Event lastTimestamp: "2026-04-24T21:49:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:49:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:07Z" name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74.18a969563b14ab41 namespace: kserve-ci-e2e-test resourceVersion: "18842" uid: 5c1c3e30-7255-4103-9d77-f52eec7a3249 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74 namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: 7425b4fb-1133-4845-89d5-240d7bb4dc45 kind: Event lastTimestamp: "2026-04-24T21:49:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:49:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:07Z" name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74.18a969563df75dc8 namespace: kserve-ci-e2e-test resourceVersion: "18844" uid: 17da9ab1-c112-40bc-bda8-917fdec3fb45 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74 namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: 7425b4fb-1133-4845-89d5-240d7bb4dc45 kind: Event lastTimestamp: "2026-04-24T21:49:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:49:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:07Z" name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74.18a969563e650f06 namespace: kserve-ci-e2e-test resourceVersion: "18845" uid: 336d55b6-c29e-4a3a-88bd-598a0cc6d4c6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T21:49:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74 namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: 7425b4fb-1133-4845-89d5-240d7bb4dc45 kind: Event lastTimestamp: "2026-04-24T21:50:05Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:49:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:05Z" name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74.18a96956aeccfce0 namespace: kserve-ci-e2e-test resourceVersion: "19258" uid: 6affa974-7679-487f-a9a5-4b75178d56e7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74 namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: 7425b4fb-1133-4845-89d5-240d7bb4dc45 kind: Event lastTimestamp: "2026-04-24T21:50:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:40Z" name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74.18a9696bf99cb3f8 namespace: kserve-ci-e2e-test resourceVersion: "19643" uid: 26ccd571-9022-476b-9837-cfd0c261ed62 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74 namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: 7425b4fb-1133-4845-89d5-240d7bb4dc45 kind: Event lastTimestamp: "2026-04-24T21:50:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:40Z" name: isvc-primary-cfc742-predictor-6f9c45779c-m9v74.18a9696bf99d4e4c namespace: kserve-ci-e2e-test resourceVersion: "19645" uid: ec4d63e3-a7fe-4300-90a7-18a5785446d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-cfc742-predictor-6f9c45779c namespace: kserve-ci-e2e-test resourceVersion: "18763" uid: 78fdde64-733c-4941-915d-75931f0b7a6c kind: Event lastTimestamp: "2026-04-24T21:49:02Z" message: 'Created pod: isvc-primary-cfc742-predictor-6f9c45779c-m9v74' metadata: creationTimestamp: "2026-04-24T21:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:02Z" name: isvc-primary-cfc742-predictor-6f9c45779c.18a969551a1a8439 namespace: kserve-ci-e2e-test resourceVersion: "18777" uid: 07bd138e-8b95-4f0a-ae16-1467dd7893ef reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-cfc742-predictor namespace: kserve-ci-e2e-test resourceVersion: "18760" uid: 5d1a8a24-7d61-4da1-b120-06478a94e9de kind: Event lastTimestamp: "2026-04-24T21:49:02Z" message: Scaled up replica set isvc-primary-cfc742-predictor-6f9c45779c from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:02Z" name: isvc-primary-cfc742-predictor.18a9695515f3f4a1 namespace: kserve-ci-e2e-test resourceVersion: "18766" uid: 35fd4e05-bd08-41b2-bea2-cdc9dfb1091b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:49:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-cfc742-predictor namespace: kserve-ci-e2e-test resourceVersion: "18762" uid: 953d55f6-a215-407e-b0c7-0434903acd2c kind: Event lastTimestamp: "2026-04-24T21:49:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:49:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:47Z" name: isvc-primary-cfc742-predictor.18a9695896516197 namespace: kserve-ci-e2e-test resourceVersion: "19134" uid: e6497dff-5299-4fc3-bf03-a20c1810b263 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:49:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-cfc742-predictor namespace: kserve-ci-e2e-test resourceVersion: "18762" uid: 953d55f6-a215-407e-b0c7-0434903acd2c kind: Event lastTimestamp: "2026-04-24T21:49:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:49:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:47Z" name: isvc-primary-cfc742-predictor.18a9695896e3b710 namespace: kserve-ci-e2e-test resourceVersion: "19135" uid: ad445d1b-bd10-4e38-9342-95672abbef51 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:50:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-cfc742-predictor namespace: kserve-ci-e2e-test resourceVersion: "18923" uid: 953d55f6-a215-407e-b0c7-0434903acd2c kind: Event lastTimestamp: "2026-04-24T21:50:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:50:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:32Z" name: isvc-primary-cfc742-predictor.18a9696314544176 namespace: kserve-ci-e2e-test resourceVersion: "19515" uid: f8af9c13-1be3-438d-a856-4d6342d6068c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:50:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-cfc742-predictor namespace: kserve-ci-e2e-test resourceVersion: "18923" uid: 953d55f6-a215-407e-b0c7-0434903acd2c kind: Event lastTimestamp: "2026-04-24T21:50:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:50:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:32Z" name: isvc-primary-cfc742-predictor.18a9696314d70bbc namespace: kserve-ci-e2e-test resourceVersion: "19516" uid: dc9b6175-24e8-47cf-a6e0-15da76397621 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T21:49:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-cfc742 namespace: kserve-ci-e2e-test resourceVersion: "18734" uid: cf2583a0-52a3-4f0f-b744-ea46049343f0 kind: Event lastTimestamp: "2026-04-24T21:50:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:15Z" name: isvc-primary-cfc742.18a9695515c833da namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: e8c7cebd-8afd-4e09-b64e-791c5dd45c5b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:49:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-cfc742 namespace: kserve-ci-e2e-test resourceVersion: "18734" uid: cf2583a0-52a3-4f0f-b744-ea46049343f0 kind: Event lastTimestamp: "2026-04-24T21:49:02Z" message: 'Failed to update status for InferenceService "isvc-primary-cfc742": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-cfc742": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:02Z" name: isvc-primary-cfc742.18a9695516285c7a namespace: kserve-ci-e2e-test resourceVersion: "18785" uid: 536c842a-8cf5-49d9-a3b0-1833de9cb730 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:49:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-cfc742 namespace: kserve-ci-e2e-test resourceVersion: "18734" uid: cf2583a0-52a3-4f0f-b744-ea46049343f0 kind: Event lastTimestamp: "2026-04-24T21:49:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-cfc742": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:02Z" name: isvc-primary-cfc742.18a9695516289c7a namespace: kserve-ci-e2e-test resourceVersion: "18786" uid: c9d028fb-f682-484c-9d0e-4783521679d5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-cfc742 namespace: kserve-ci-e2e-test resourceVersion: "19335" uid: cf2583a0-52a3-4f0f-b744-ea46049343f0 kind: Event lastTimestamp: "2026-04-24T21:50:15Z" message: InferenceService [isvc-primary-cfc742] is Ready metadata: creationTimestamp: "2026-04-24T21:50:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:15Z" name: isvc-primary-cfc742.18a96966128b525e namespace: kserve-ci-e2e-test resourceVersion: "19336" uid: 63207781-b592-403d-8d5f-81858079735c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:40:00.962262Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg to ip-10-0-134-249.ec2.internal metadata: creationTimestamp: "2026-04-24T21:40:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:40:00Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968d715845933 namespace: kserve-ci-e2e-test resourceVersion: "13991" uid: 49a76207-f034-4582-8a9c-53ff833d9fc9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6688756845-wvlxd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13997" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:40:01Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:40:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:40:01Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968d72eac00e5 namespace: kserve-ci-e2e-test resourceVersion: "14008" uid: 7fa31e33-4676-488a-be36-dd7ce3ba7435 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:40:01Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0dd2798f9cdbffc0563bd148612201df0e589ff8c26a6b19d321fd120fc5c097" metadata: creationTimestamp: "2026-04-24T21:40:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:01Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968d730421884 namespace: kserve-ci-e2e-test resourceVersion: "14010" uid: cb81631e-eb03-4cbd-b4c8-1c021fc412eb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:40:04Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0dd2798f9cdbffc0563bd148612201df0e589ff8c26a6b19d321fd120fc5c097" in 3.179s (3.179s including waiting). Image size: 299845049 bytes.' metadata: creationTimestamp: "2026-04-24T21:40:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:04Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968d7edbf1db4 namespace: kserve-ci-e2e-test resourceVersion: "14037" uid: 17f66642-c141-42c4-b0b4-05ab375fa3bc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:40:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:40:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:04Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968d7f131f33f namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: ef2d453f-17d9-4332-9f71-70cab4540d3b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:40:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:40:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:04Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968d7f1a217e6 namespace: kserve-ci-e2e-test resourceVersion: "14039" uid: 38ef8155-a604-47ab-bde1-c8f732a44680 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:40:09Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1450" metadata: creationTimestamp: "2026-04-24T21:40:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:09Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968d8f9cfcd94 namespace: kserve-ci-e2e-test resourceVersion: "14072" uid: a44e68bf-03d0-4c66-a9b6-8774d1116578 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:40:22Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1450" in 13.138s (13.138s including waiting). Image size: 1560926126 bytes.' metadata: creationTimestamp: "2026-04-24T21:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:22Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968dc08edbdd1 namespace: kserve-ci-e2e-test resourceVersion: "14173" uid: 53f81bef-4129-49b1-8e0b-c5c00dbe8849 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:40:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:22Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968dc0c3bccfd namespace: kserve-ci-e2e-test resourceVersion: "14174" uid: ddbf5690-a22f-46ab-88a6-cc74b614f65c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:40:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:22Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968dc0caa148d namespace: kserve-ci-e2e-test resourceVersion: "14175" uid: abf8456a-071a-4879-9836-0ee0c53a6004 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:40:22Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T21:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:22Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968dc0cb4cf39 namespace: kserve-ci-e2e-test resourceVersion: "14176" uid: dc679c9c-4950-4f14-9101-63275e5478bc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:40:24Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.21s (2.21s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T21:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:24Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968dc907e017c namespace: kserve-ci-e2e-test resourceVersion: "14195" uid: b346796a-6f3b-4ca3-9679-7e6fdaff22fc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:40:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:24Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968dc93c79da8 namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: fe41afbf-3533-426b-a0d1-74e65ba77883 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:40:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:24Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968dc9435b8c7 namespace: kserve-ci-e2e-test resourceVersion: "14197" uid: e421f93c-c2e9-4c85-a9cc-9d252002398c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:40:24Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:f72c005cb705ef76fbeae81cf78e50cb05f1674ac01bdd8893cf8fb48213f3d0" metadata: creationTimestamp: "2026-04-24T21:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:24Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968dc9441ec1b namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: f6d7f24d-8d95-4a35-964e-d755a9bef368 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:40:27Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:f72c005cb705ef76fbeae81cf78e50cb05f1674ac01bdd8893cf8fb48213f3d0" in 2.975s (2.975s including waiting). Image size: 238035033 bytes.' metadata: creationTimestamp: "2026-04-24T21:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:27Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968dd459a2152 namespace: kserve-ci-e2e-test resourceVersion: "14215" uid: 58caf30b-c604-4a2c-a364-98abdaba9d77 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:40:27Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-24T21:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:27Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968dd48b8cdfe namespace: kserve-ci-e2e-test resourceVersion: "14217" uid: 8e38e2db-bd8a-46aa-8bc4-96b97fa61847 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:40:27Z" message: Started container agent metadata: creationTimestamp: "2026-04-24T21:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:27Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968dd492a0bf3 namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: 13f7509f-d6fa-4e32-b87d-64d485a56bef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T21:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:42:01Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:42:01Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968dda709c1f9 namespace: kserve-ci-e2e-test resourceVersion: "15052" uid: ae726659-2b1d-43ad-8a5b-3c7aa150b972 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T21:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:42:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:42:01Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968dda712eb65 namespace: kserve-ci-e2e-test resourceVersion: "15053" uid: d77ddad6-9e53-4e87-8063-78c3c43e69f1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:41:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:45Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968ef8839f299 namespace: kserve-ci-e2e-test resourceVersion: "14820" uid: c733e083-5115-4380-b4a5-fe0bc42eff19 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:41:45Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-24T21:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:45Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968ef883c4e8b namespace: kserve-ci-e2e-test resourceVersion: "14822" uid: fd502948-6aaa-4604-852b-e74e662be338 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:41:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:45Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968ef883d679b namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: ec8a7d7e-3adc-4d49-86ca-eee6d460a8a8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:41:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: b9e83759-d6d0-4ac8-8fd5-07b71bd7a32e kind: Event lastTimestamp: "2026-04-24T21:42:09Z" message: 'Readiness probe failed: Get "https://10.132.0.34:8643/healthz": dial tcp 10.132.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:42:09Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg.18a968f04757bfdd namespace: kserve-ci-e2e-test resourceVersion: "15110" uid: f02e6e3a-52b6-4f73-9aa9-865345c3f4ae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8 namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: 763fe9da-d3b3-40c6-bdec-e95798eea0ac kind: Event lastTimestamp: "2026-04-24T21:40:00Z" message: 'Created pod: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8-h6drg' metadata: creationTimestamp: "2026-04-24T21:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:00Z" name: isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8.18a968d714daca35 namespace: kserve-ci-e2e-test resourceVersion: "13987" uid: 9d319f2f-296c-4b0b-98f0-83b54683afc4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-ae801-predictor namespace: kserve-ci-e2e-test resourceVersion: "13977" uid: 77e16e81-89a9-438d-9a52-9d521b5a0030 kind: Event lastTimestamp: "2026-04-24T21:40:00Z" message: Scaled up replica set isvc-raw-sklearn-batcher-ae801-predictor-74c8899fd8 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:00Z" name: isvc-raw-sklearn-batcher-ae801-predictor.18a968d70ee0481b namespace: kserve-ci-e2e-test resourceVersion: "13979" uid: 98373e7e-8eb7-4196-a58c-ab9b6949f0ed reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:40:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-ae801-predictor namespace: kserve-ci-e2e-test resourceVersion: "13986" uid: fbcb0a0d-06e0-4e06-a9aa-defeb3a3c8db kind: Event lastTimestamp: "2026-04-24T21:41:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:40:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:01Z" name: isvc-raw-sklearn-batcher-ae801-predictor.18a968da948fc09c namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: da81819e-6407-44ce-9f82-1366f9a410f1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:40:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-ae801-predictor namespace: kserve-ci-e2e-test resourceVersion: "13986" uid: fbcb0a0d-06e0-4e06-a9aa-defeb3a3c8db kind: Event lastTimestamp: "2026-04-24T21:41:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:40:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:01Z" name: isvc-raw-sklearn-batcher-ae801-predictor.18a968da9508705f namespace: kserve-ci-e2e-test resourceVersion: "14460" uid: ebb5d49d-bd88-41fa-9600-ff9139e820a0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:41:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-ae801-predictor namespace: kserve-ci-e2e-test resourceVersion: "14131" uid: fbcb0a0d-06e0-4e06-a9aa-defeb3a3c8db kind: Event lastTimestamp: "2026-04-24T21:41:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:41:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:31Z" name: isvc-raw-sklearn-batcher-ae801-predictor.18a968e891ac44b5 namespace: kserve-ci-e2e-test resourceVersion: "14679" uid: 170d86fd-588e-4bd8-81be-ee8c8999f690 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:41:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-ae801-predictor namespace: kserve-ci-e2e-test resourceVersion: "14131" uid: fbcb0a0d-06e0-4e06-a9aa-defeb3a3c8db kind: Event lastTimestamp: "2026-04-24T21:41:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:41:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:31Z" name: isvc-raw-sklearn-batcher-ae801-predictor.18a968e89222b5fb namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: c9e3ba35-b8cb-464f-9fee-5ac313d006d1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-24T21:40:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-ae801 namespace: kserve-ci-e2e-test resourceVersion: "13965" uid: f01f329a-35ba-4c78-ac27-b65fa9e6d146 kind: Event lastTimestamp: "2026-04-24T21:41:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:41:31Z" name: isvc-raw-sklearn-batcher-ae801.18a968d71514530a namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: 460e527f-2e96-4829-807d-2bd3a23ccda6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-ae801 namespace: kserve-ci-e2e-test resourceVersion: "13965" uid: f01f329a-35ba-4c78-ac27-b65fa9e6d146 kind: Event lastTimestamp: "2026-04-24T21:40:00Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-ae801": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-ae801": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:40:00Z" name: isvc-raw-sklearn-batcher-ae801.18a968d7159fef0d namespace: kserve-ci-e2e-test resourceVersion: "13993" uid: 80afcb8e-ce7f-4dff-b4db-a5430ea6a46f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-ae801 namespace: kserve-ci-e2e-test resourceVersion: "13965" uid: f01f329a-35ba-4c78-ac27-b65fa9e6d146 kind: Event lastTimestamp: "2026-04-24T21:40:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-ae801": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:40:00Z" name: isvc-raw-sklearn-batcher-ae801.18a968d715a02ef6 namespace: kserve-ci-e2e-test resourceVersion: "13995" uid: 437e93e2-d9ee-4c51-8c54-a716361e5ee9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-ae801 namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: f01f329a-35ba-4c78-ac27-b65fa9e6d146 kind: Event lastTimestamp: "2026-04-24T21:41:31Z" message: InferenceService [isvc-raw-sklearn-batcher-ae801] is Ready metadata: creationTimestamp: "2026-04-24T21:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:41:31Z" name: isvc-raw-sklearn-batcher-ae801.18a968ec1cc052ad namespace: kserve-ci-e2e-test resourceVersion: "14689" uid: b65b49e1-3938-4fe3-b66e-d202a4ae9fe1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:50:22.320467Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-cfc742-predictor-68cdb59558-vm6gb namespace: kserve-ci-e2e-test resourceVersion: "19414" uid: fbc34c10-26df-40e0-8b89-0615b9911212 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-cfc742-predictor-68cdb59558-vm6gb to ip-10-0-134-249.ec2.internal metadata: creationTimestamp: "2026-04-24T21:50:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:50:22Z" name: isvc-secondary-cfc742-predictor-68cdb59558-vm6gb.18a96967c154bcb0 namespace: kserve-ci-e2e-test resourceVersion: "19418" uid: 81bc4370-7719-4785-9b23-75340bb3c46f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6688756845-wvlxd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:22Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-cfc742-predictor-68cdb59558-vm6gb namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: fbc34c10-26df-40e0-8b89-0615b9911212 kind: Event lastTimestamp: "2026-04-24T21:50:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-secondary-cfc742-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:22Z" name: isvc-secondary-cfc742-predictor-68cdb59558-vm6gb.18a96967c806b3f0 namespace: kserve-ci-e2e-test resourceVersion: "19427" uid: 628b228c-2367-41fa-874a-34a671d410a9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:23Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-cfc742-predictor-68cdb59558-vm6gb namespace: kserve-ci-e2e-test resourceVersion: "19423" uid: fbc34c10-26df-40e0-8b89-0615b9911212 kind: Event lastTimestamp: "2026-04-24T21:50:23Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:50:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:50:23Z" name: isvc-secondary-cfc742-predictor-68cdb59558-vm6gb.18a96967fe0b58c9 namespace: kserve-ci-e2e-test resourceVersion: "19437" uid: ab1519a2-1bec-40b6-a3ce-3b3719a94f93 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:50:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-cfc742-predictor-68cdb59558-vm6gb namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: fbc34c10-26df-40e0-8b89-0615b9911212 kind: Event lastTimestamp: "2026-04-24T21:50:29Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0dd2798f9cdbffc0563bd148612201df0e589ff8c26a6b19d321fd120fc5c097" already present on machine metadata: creationTimestamp: "2026-04-24T21:50:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:29Z" name: isvc-secondary-cfc742-predictor-68cdb59558-vm6gb.18a96967ff7db837 namespace: kserve-ci-e2e-test resourceVersion: "19489" uid: 9c06ed91-7078-4211-9298-be12e592d75e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:50:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-cfc742-predictor-68cdb59558-vm6gb namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: fbc34c10-26df-40e0-8b89-0615b9911212 kind: Event lastTimestamp: "2026-04-24T21:50:29Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:50:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:29Z" name: isvc-secondary-cfc742-predictor-68cdb59558-vm6gb.18a969680242d452 namespace: kserve-ci-e2e-test resourceVersion: "19493" uid: 25f0ac81-1060-47fc-aa32-1b7e54c1f275 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:50:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-cfc742-predictor-68cdb59558-vm6gb namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: fbc34c10-26df-40e0-8b89-0615b9911212 kind: Event lastTimestamp: "2026-04-24T21:50:29Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:50:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:29Z" name: isvc-secondary-cfc742-predictor-68cdb59558-vm6gb.18a9696802b01343 namespace: kserve-ci-e2e-test resourceVersion: "19494" uid: 361cbda5-d05d-4bfc-9386-e95481ef6a20 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-cfc742-predictor-68cdb59558-vm6gb namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: fbc34c10-26df-40e0-8b89-0615b9911212 kind: Event lastTimestamp: "2026-04-24T21:50:35Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-cfc742-predictor-68cdb59558-vm6gb_kserve-ci-e2e-test(fbc34c10-26df-40e0-8b89-0615b9911212) metadata: creationTimestamp: "2026-04-24T21:50:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:35Z" name: isvc-secondary-cfc742-predictor-68cdb59558-vm6gb.18a9696ac71f7a4d namespace: kserve-ci-e2e-test resourceVersion: "19543" uid: 96c95878-f52b-4a2b-b129-9365b6a1cc89 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-cfc742-predictor-68cdb59558 namespace: kserve-ci-e2e-test resourceVersion: "19405" uid: 4c0ce112-56ee-4a44-a4cf-43fff033b991 kind: Event lastTimestamp: "2026-04-24T21:50:22Z" message: 'Created pod: isvc-secondary-cfc742-predictor-68cdb59558-vm6gb' metadata: creationTimestamp: "2026-04-24T21:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:22Z" name: isvc-secondary-cfc742-predictor-68cdb59558.18a96967c0e8b083 namespace: kserve-ci-e2e-test resourceVersion: "19415" uid: 746b347e-c1f1-419b-9bba-aaa4d4fdcb80 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-cfc742-predictor namespace: kserve-ci-e2e-test resourceVersion: "19403" uid: cc2e05e7-2643-436b-8a3b-5027d4b3e524 kind: Event lastTimestamp: "2026-04-24T21:50:22Z" message: Scaled up replica set isvc-secondary-cfc742-predictor-68cdb59558 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:22Z" name: isvc-secondary-cfc742-predictor.18a96967bdd2b318 namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: 86141823-d8fc-4b93-976a-0c92c094f6be reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-cfc742-predictor namespace: kserve-ci-e2e-test resourceVersion: "19404" uid: 56f97015-38a7-45a6-a707-dda52892dacf kind: Event lastTimestamp: "2026-04-24T21:50:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:37Z" name: isvc-secondary-cfc742-predictor.18a9696b3c626737 namespace: kserve-ci-e2e-test resourceVersion: "19564" uid: d7f29b05-5e18-432f-ae4e-bd3638f852aa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-cfc742-predictor namespace: kserve-ci-e2e-test resourceVersion: "19404" uid: 56f97015-38a7-45a6-a707-dda52892dacf kind: Event lastTimestamp: "2026-04-24T21:50:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:37Z" name: isvc-secondary-cfc742-predictor.18a9696b3cd508c6 namespace: kserve-ci-e2e-test resourceVersion: "19566" uid: 9f38429b-9024-4891-8f37-363e28b78e26 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-24T21:50:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-cfc742 namespace: kserve-ci-e2e-test resourceVersion: "19394" uid: 5dfef760-cf5e-4aa6-90d4-070da5c8208c kind: Event lastTimestamp: "2026-04-24T21:50:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:35Z" name: isvc-secondary-cfc742.18a96967bda760c1 namespace: kserve-ci-e2e-test resourceVersion: "19552" uid: 5468eea9-500a-47c0-b5a0-8cae91f80d50 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-cfc742 namespace: kserve-ci-e2e-test resourceVersion: "19394" uid: 5dfef760-cf5e-4aa6-90d4-070da5c8208c kind: Event lastTimestamp: "2026-04-24T21:50:22Z" message: 'Failed to update status for InferenceService "isvc-secondary-cfc742": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-cfc742": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:22Z" name: isvc-secondary-cfc742.18a96967be027397 namespace: kserve-ci-e2e-test resourceVersion: "19409" uid: 497a31c2-738f-448e-94a5-58ace21ebae8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-cfc742 namespace: kserve-ci-e2e-test resourceVersion: "19394" uid: 5dfef760-cf5e-4aa6-90d4-070da5c8208c kind: Event lastTimestamp: "2026-04-24T21:50:22Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-cfc742": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:22Z" name: isvc-secondary-cfc742.18a96967be02b6cd namespace: kserve-ci-e2e-test resourceVersion: "19410" uid: 19d9d69d-3181-49e8-8021-ea1598638949 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:41:46.067107Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6 namespace: kserve-ci-e2e-test resourceVersion: "14842" uid: f7636b02-9417-4eed-90f5-a4d603191037 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6 to ip-10-0-134-249.ec2.internal metadata: creationTimestamp: "2026-04-24T21:41:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:41:46Z" name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6.18a968ef8e3b8e81 namespace: kserve-ci-e2e-test resourceVersion: "14855" uid: bc7c945f-bd93-4c42-9db6-f962e82b01cc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6688756845-wvlxd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:46Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6 namespace: kserve-ci-e2e-test resourceVersion: "14861" uid: f7636b02-9417-4eed-90f5-a4d603191037 kind: Event lastTimestamp: "2026-04-24T21:41:46Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:41:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:41:46Z" name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6.18a968efb3b3ada3 namespace: kserve-ci-e2e-test resourceVersion: "14887" uid: 8ba3c339-0eb1-403f-af8c-2f9a2f3690eb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6 namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: f7636b02-9417-4eed-90f5-a4d603191037 kind: Event lastTimestamp: "2026-04-24T21:41:46Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0dd2798f9cdbffc0563bd148612201df0e589ff8c26a6b19d321fd120fc5c097" already present on machine metadata: creationTimestamp: "2026-04-24T21:41:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:46Z" name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6.18a968efb5407e89 namespace: kserve-ci-e2e-test resourceVersion: "14890" uid: 26325637-042d-46c3-a15d-c2504723d257 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6 namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: f7636b02-9417-4eed-90f5-a4d603191037 kind: Event lastTimestamp: "2026-04-24T21:41:46Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:41:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:46Z" name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6.18a968efb7fceec2 namespace: kserve-ci-e2e-test resourceVersion: "14893" uid: bdd32330-83af-4d83-bd79-3ad714ed8f3c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6 namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: f7636b02-9417-4eed-90f5-a4d603191037 kind: Event lastTimestamp: "2026-04-24T21:41:46Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:41:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:46Z" name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6.18a968efb86c6102 namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: ac4c154b-03e9-42cd-90d9-877258c003b5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6 namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: f7636b02-9417-4eed-90f5-a4d603191037 kind: Event lastTimestamp: "2026-04-24T21:41:50Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:50Z" name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6.18a968f094144ed0 namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: 5b6c37ad-b233-44a5-9a33-f579a1037388 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6 namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: f7636b02-9417-4eed-90f5-a4d603191037 kind: Event lastTimestamp: "2026-04-24T21:41:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:50Z" name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6.18a968f098dc33ac namespace: kserve-ci-e2e-test resourceVersion: "14945" uid: f66c4f2b-dece-456b-8cc4-563ad547efe5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6 namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: f7636b02-9417-4eed-90f5-a4d603191037 kind: Event lastTimestamp: "2026-04-24T21:41:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:50Z" name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6.18a968f0994d7cb4 namespace: kserve-ci-e2e-test resourceVersion: "14947" uid: 17024585-4c64-435a-a726-66c2b0a62dbc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6 namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: f7636b02-9417-4eed-90f5-a4d603191037 kind: Event lastTimestamp: "2026-04-24T21:41:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:50Z" name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6.18a968f0995ad145 namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: c08c33e2-d9e3-418d-b330-eb6105b71499 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6 namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: f7636b02-9417-4eed-90f5-a4d603191037 kind: Event lastTimestamp: "2026-04-24T21:41:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:50Z" name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6.18a968f09d052974 namespace: kserve-ci-e2e-test resourceVersion: "14950" uid: 44df1b07-079e-4185-9337-6bd904c9daeb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6 namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: f7636b02-9417-4eed-90f5-a4d603191037 kind: Event lastTimestamp: "2026-04-24T21:41:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:50Z" name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6.18a968f09d7070eb namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: b0822167-cb62-4ab4-b0e5-b97820e7280f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:41:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6 namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: f7636b02-9417-4eed-90f5-a4d603191037 kind: Event lastTimestamp: "2026-04-24T21:43:28Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:28Z" name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6.18a968f10c13b169 namespace: kserve-ci-e2e-test resourceVersion: "15907" uid: ac842230-c5c4-4f58-b779-dfd1e7c263f0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6 namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: f7636b02-9417-4eed-90f5-a4d603191037 kind: Event lastTimestamp: "2026-04-24T21:43:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:26Z" name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6.18a96906e3dd49dc namespace: kserve-ci-e2e-test resourceVersion: "15795" uid: 50951486-1950-4d5f-9310-c2bacf9b2b34 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6 namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: f7636b02-9417-4eed-90f5-a4d603191037 kind: Event lastTimestamp: "2026-04-24T21:43:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:26Z" name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6.18a96906e3de2366 namespace: kserve-ci-e2e-test resourceVersion: "15798" uid: c8e8f3c5-22d1-4bc7-91c7-3c04f6f7fc10 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6 namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: f7636b02-9417-4eed-90f5-a4d603191037 kind: Event lastTimestamp: "2026-04-24T21:43:28Z" message: 'Readiness probe failed: Get "https://10.132.0.35:8643/healthz": dial tcp 10.132.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:28Z" name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6.18a96907664c3b6c namespace: kserve-ci-e2e-test resourceVersion: "15905" uid: 43eb2af9-0ba9-4f67-8ac1-aa08cad49ed0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96 namespace: kserve-ci-e2e-test resourceVersion: "14823" uid: dc9ce98f-5adf-4682-9eed-4528d10452ed kind: Event lastTimestamp: "2026-04-24T21:41:46Z" message: 'Created pod: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96-kx8l6' metadata: creationTimestamp: "2026-04-24T21:41:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:46Z" name: isvc-sklearn-graph-raw-62689-predictor-6fdc5df96.18a968ef8d6bef36 namespace: kserve-ci-e2e-test resourceVersion: "14848" uid: c1f3af55-ee44-4798-ae44-3dcc0a708f1a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-62689-predictor namespace: kserve-ci-e2e-test resourceVersion: "14819" uid: e5de12c6-109a-47aa-90a1-07322070f91b kind: Event lastTimestamp: "2026-04-24T21:41:45Z" message: Scaled up replica set isvc-sklearn-graph-raw-62689-predictor-6fdc5df96 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:45Z" name: isvc-sklearn-graph-raw-62689-predictor.18a968ef892e286c namespace: kserve-ci-e2e-test resourceVersion: "14825" uid: 884293b3-b327-4390-87aa-c852d1a5a89c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:42:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-62689-predictor namespace: kserve-ci-e2e-test resourceVersion: "14821" uid: 2f80ad0f-bf5a-4c14-9cb4-064a698fecf3 kind: Event lastTimestamp: "2026-04-24T21:42:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:42:31Z" name: isvc-sklearn-graph-raw-62689-predictor.18a968f3087b927e namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: 443113b7-e84f-4751-92ba-90ce56002d16 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:42:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-62689-predictor namespace: kserve-ci-e2e-test resourceVersion: "14821" uid: 2f80ad0f-bf5a-4c14-9cb4-064a698fecf3 kind: Event lastTimestamp: "2026-04-24T21:42:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:42:31Z" name: isvc-sklearn-graph-raw-62689-predictor.18a968f308e91a51 namespace: kserve-ci-e2e-test resourceVersion: "15284" uid: 6a84495c-2d4d-443f-96fd-ec4dcccb26a2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:42:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-62689-predictor namespace: kserve-ci-e2e-test resourceVersion: "15046" uid: 2f80ad0f-bf5a-4c14-9cb4-064a698fecf3 kind: Event lastTimestamp: "2026-04-24T21:43:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:43:16Z" name: isvc-sklearn-graph-raw-62689-predictor.18a968f687a3d982 namespace: kserve-ci-e2e-test resourceVersion: "15647" uid: 8d39b242-0f2a-463b-858c-780e7cc5469f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:42:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-62689-predictor namespace: kserve-ci-e2e-test resourceVersion: "15046" uid: 2f80ad0f-bf5a-4c14-9cb4-064a698fecf3 kind: Event lastTimestamp: "2026-04-24T21:43:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:43:16Z" name: isvc-sklearn-graph-raw-62689-predictor.18a968f688134063 namespace: kserve-ci-e2e-test resourceVersion: "15649" uid: d2265221-5e68-4973-a94e-0a495cfc011d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T21:41:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-62689 namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 43b035fd-ad31-45ac-9734-2bc4caf28bd6 kind: Event lastTimestamp: "2026-04-24T21:42:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:42:58Z" name: isvc-sklearn-graph-raw-62689.18a968ef8902f18b namespace: kserve-ci-e2e-test resourceVersion: "15492" uid: 5f9db1d8-4a2f-409e-b1c1-33b8860aa3e6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-62689 namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 43b035fd-ad31-45ac-9734-2bc4caf28bd6 kind: Event lastTimestamp: "2026-04-24T21:41:45Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-62689": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-62689": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:41:45Z" name: isvc-sklearn-graph-raw-62689.18a968ef895fa7b5 namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: 564a91ef-ccc0-4248-97f6-02dc0d8ebb6c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:42:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-62689 namespace: kserve-ci-e2e-test resourceVersion: "15490" uid: 43b035fd-ad31-45ac-9734-2bc4caf28bd6 kind: Event lastTimestamp: "2026-04-24T21:42:58Z" message: InferenceService [isvc-sklearn-graph-raw-62689] is Ready metadata: creationTimestamp: "2026-04-24T21:42:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:42:58Z" name: isvc-sklearn-graph-raw-62689.18a969006f8c9048 namespace: kserve-ci-e2e-test resourceVersion: "15491" uid: 23dda4d7-b454-4e4d-b633-183c15431cf6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:43:26.347757Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94 namespace: kserve-ci-e2e-test resourceVersion: "15803" uid: 0c696334-2346-49ce-8d0b-8991eb2436e4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94 to ip-10-0-134-249.ec2.internal metadata: creationTimestamp: "2026-04-24T21:43:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:43:26Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94.18a96906e76cef6e namespace: kserve-ci-e2e-test resourceVersion: "15813" uid: bbde6c4c-01ee-4e75-aef7-e7599b985d9e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6688756845-wvlxd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:26Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: 0c696334-2346-49ce-8d0b-8991eb2436e4 kind: Event lastTimestamp: "2026-04-24T21:43:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-hpa-2b4f4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:26Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94.18a96906f1fde44f namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: a0ca0d73-ef63-48df-a906-07a1431ffbdd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94 namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: 0c696334-2346-49ce-8d0b-8991eb2436e4 kind: Event lastTimestamp: "2026-04-24T21:43:27Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:43:27Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94.18a9690724be3489 namespace: kserve-ci-e2e-test resourceVersion: "15881" uid: c9e00ee0-4706-4c4f-8ca1-c007c40feff3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: 0c696334-2346-49ce-8d0b-8991eb2436e4 kind: Event lastTimestamp: "2026-04-24T21:43:27Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0dd2798f9cdbffc0563bd148612201df0e589ff8c26a6b19d321fd120fc5c097" already present on machine metadata: creationTimestamp: "2026-04-24T21:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:27Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94.18a96907262bbd37 namespace: kserve-ci-e2e-test resourceVersion: "15883" uid: c55fbc87-899b-4a0d-a5ef-e4acc7fede91 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: 0c696334-2346-49ce-8d0b-8991eb2436e4 kind: Event lastTimestamp: "2026-04-24T21:43:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:27Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94.18a9690729b835b6 namespace: kserve-ci-e2e-test resourceVersion: "15884" uid: 34fdfc12-16d6-4663-b34a-5e23fca84b55 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: 0c696334-2346-49ce-8d0b-8991eb2436e4 kind: Event lastTimestamp: "2026-04-24T21:43:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:27Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94.18a969072a26b2fa namespace: kserve-ci-e2e-test resourceVersion: "15885" uid: c07d0c40-5626-4840-bf0e-4b0bae408b38 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: 0c696334-2346-49ce-8d0b-8991eb2436e4 kind: Event lastTimestamp: "2026-04-24T21:43:31Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:31Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94.18a969082ecf6297 namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: e5e82624-93d9-4463-acae-07f03511d050 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: 0c696334-2346-49ce-8d0b-8991eb2436e4 kind: Event lastTimestamp: "2026-04-24T21:43:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:31Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94.18a96908321e7101 namespace: kserve-ci-e2e-test resourceVersion: "15956" uid: 812bb578-e47a-423d-bf1d-4c52beecc143 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: 0c696334-2346-49ce-8d0b-8991eb2436e4 kind: Event lastTimestamp: "2026-04-24T21:43:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:31Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94.18a96908328e1c2e namespace: kserve-ci-e2e-test resourceVersion: "15957" uid: 85c986b5-108c-4f52-a9bf-174f89526459 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: 0c696334-2346-49ce-8d0b-8991eb2436e4 kind: Event lastTimestamp: "2026-04-24T21:43:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:31Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94.18a969083298e3d2 namespace: kserve-ci-e2e-test resourceVersion: "15958" uid: b2a3aa7a-c8e2-43c0-8c74-5a49838d2e49 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: 0c696334-2346-49ce-8d0b-8991eb2436e4 kind: Event lastTimestamp: "2026-04-24T21:43:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:31Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94.18a9690835c41480 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: ff465a0c-6a92-4bc6-90d1-716f550db272 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: 0c696334-2346-49ce-8d0b-8991eb2436e4 kind: Event lastTimestamp: "2026-04-24T21:43:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:31Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94.18a969083632706b namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 93b2c3f3-f254-4091-8da6-f1901e8a5511 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:43:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: 0c696334-2346-49ce-8d0b-8991eb2436e4 kind: Event lastTimestamp: "2026-04-24T21:45:09Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:09Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94.18a96908a6a732a8 namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: ada84f0a-d9a6-45cf-ac7a-663e2135efb0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: 0c696334-2346-49ce-8d0b-8991eb2436e4 kind: Event lastTimestamp: "2026-04-24T21:45:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:45:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:06Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94.18a9691e3e86d7c9 namespace: kserve-ci-e2e-test resourceVersion: "16780" uid: fc706885-4a76-4be5-b376-a18c09329fe1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: 0c696334-2346-49ce-8d0b-8991eb2436e4 kind: Event lastTimestamp: "2026-04-24T21:45:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:45:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:06Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94.18a9691e3e875a89 namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 31d09b3a-40f0-4e98-bfe9-003221c5c9b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: 0c696334-2346-49ce-8d0b-8991eb2436e4 kind: Event lastTimestamp: "2026-04-24T21:45:09Z" message: 'Readiness probe failed: Get "https://10.132.0.37:8643/healthz": dial tcp 10.132.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:45:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:09Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94.18a9691f00d1059d namespace: kserve-ci-e2e-test resourceVersion: "16857" uid: 40bf1a23-fd1e-4aae-9f71-50e5c6f72f69 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968 namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: fd74cb30-5677-4ef1-8224-a59e8f989b7f kind: Event lastTimestamp: "2026-04-24T21:43:26Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968-6jf94' metadata: creationTimestamp: "2026-04-24T21:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:43:26Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968.18a96906e6eab55c namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: acd14693-3de9-4aea-bb65-6a8b9a28c659 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: 03598759-72fc-4bdb-9aac-e06f949be2a4 kind: Event lastTimestamp: "2026-04-24T21:43:26Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-2b4f4-predictor-8485789968 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:43:26Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor.18a96906e2385fb7 namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: adf59711-3913-4ee7-ab6f-d38d3c56abde reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15776" uid: 0611649a-e6cb-4537-bd51-6cd2d666de17 kind: Event lastTimestamp: "2026-04-24T21:43:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:43:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:43:41Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor.18a9690a60ddaa34 namespace: kserve-ci-e2e-test resourceVersion: "16045" uid: f361d6a8-6071-465d-9617-52b875531583 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15776" uid: 0611649a-e6cb-4537-bd51-6cd2d666de17 kind: Event lastTimestamp: "2026-04-24T21:43:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:43:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:43:41Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor.18a9690a6159fbb1 namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 4bfbb610-aaa0-4146-b7b8-a657ca63195b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:43:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: 0611649a-e6cb-4537-bd51-6cd2d666de17 kind: Event lastTimestamp: "2026-04-24T21:44:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:44:56Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor.18a9690de0a13bf6 namespace: kserve-ci-e2e-test resourceVersion: "16621" uid: e5cdf07f-42c3-46bc-9650-c6f52f8fdc1c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:43:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: 0611649a-e6cb-4537-bd51-6cd2d666de17 kind: Event lastTimestamp: "2026-04-24T21:44:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:44:56Z" name: isvc-sklearn-graph-raw-hpa-2b4f4-predictor.18a9690de125600a namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: 753c5863-f9ec-487a-8eb4-c3a5e6a76b1f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:43:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-2b4f4 namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: b0cd48ca-9c76-4ec9-9618-e4170ed992a8 kind: Event lastTimestamp: "2026-04-24T21:44:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:44:39Z" name: isvc-sklearn-graph-raw-hpa-2b4f4.18a96906e212833c namespace: kserve-ci-e2e-test resourceVersion: "16493" uid: 365b9f73-58a4-499d-a791-048420a05d19 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-2b4f4 namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: b0cd48ca-9c76-4ec9-9618-e4170ed992a8 kind: Event lastTimestamp: "2026-04-24T21:43:26Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-2b4f4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-2b4f4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:43:26Z" name: isvc-sklearn-graph-raw-hpa-2b4f4.18a96906e30c5cb3 namespace: kserve-ci-e2e-test resourceVersion: "15789" uid: ea167af8-5750-4ab9-a20d-53545a0b0023 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:44:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-2b4f4 namespace: kserve-ci-e2e-test resourceVersion: "16490" uid: b0cd48ca-9c76-4ec9-9618-e4170ed992a8 kind: Event lastTimestamp: "2026-04-24T21:44:39Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-2b4f4] is Ready metadata: creationTimestamp: "2026-04-24T21:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:44:39Z" name: isvc-sklearn-graph-raw-hpa-2b4f4.18a9691809db8942 namespace: kserve-ci-e2e-test resourceVersion: "16491" uid: 03efd180-33d7-46f5-8c38-b1a8104a814b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:46:41.908896Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5 namespace: kserve-ci-e2e-test resourceVersion: "17666" uid: f9175222-2aa6-4fc1-90f3-9e45fe75afc3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5 to ip-10-0-134-249.ec2.internal metadata: creationTimestamp: "2026-04-24T21:46:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:46:41Z" name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5.18a969346fc711f3 namespace: kserve-ci-e2e-test resourceVersion: "17673" uid: 19f46ccc-42be-4c5d-8779-d3b27a89bd1a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6688756845-wvlxd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:42Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5 namespace: kserve-ci-e2e-test resourceVersion: "17671" uid: f9175222-2aa6-4fc1-90f3-9e45fe75afc3 kind: Event lastTimestamp: "2026-04-24T21:46:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-e8206-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:46:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:42Z" name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5.18a9693478587a19 namespace: kserve-ci-e2e-test resourceVersion: "17697" uid: 9d05e89e-b641-45bb-93ce-358784c142f2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:42Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5 namespace: kserve-ci-e2e-test resourceVersion: "17677" uid: f9175222-2aa6-4fc1-90f3-9e45fe75afc3 kind: Event lastTimestamp: "2026-04-24T21:46:42Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:46:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:46:42Z" name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5.18a96934ac544412 namespace: kserve-ci-e2e-test resourceVersion: "17712" uid: 0bc5c61c-b233-4456-b796-118f1d2786b2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5 namespace: kserve-ci-e2e-test resourceVersion: "17671" uid: f9175222-2aa6-4fc1-90f3-9e45fe75afc3 kind: Event lastTimestamp: "2026-04-24T21:46:42Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0dd2798f9cdbffc0563bd148612201df0e589ff8c26a6b19d321fd120fc5c097" already present on machine metadata: creationTimestamp: "2026-04-24T21:46:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:42Z" name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5.18a96934adc07770 namespace: kserve-ci-e2e-test resourceVersion: "17714" uid: 8bc0ef86-d41a-4ba4-9c71-81eb550c4411 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5 namespace: kserve-ci-e2e-test resourceVersion: "17671" uid: f9175222-2aa6-4fc1-90f3-9e45fe75afc3 kind: Event lastTimestamp: "2026-04-24T21:46:42Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:46:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:43Z" name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5.18a96934b0cef830 namespace: kserve-ci-e2e-test resourceVersion: "17716" uid: 98f559ea-9c00-4a7a-9810-140de5e8e4e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5 namespace: kserve-ci-e2e-test resourceVersion: "17671" uid: f9175222-2aa6-4fc1-90f3-9e45fe75afc3 kind: Event lastTimestamp: "2026-04-24T21:46:43Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:46:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:43Z" name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5.18a96934b13b024e namespace: kserve-ci-e2e-test resourceVersion: "17717" uid: e6ea3375-22db-4c85-9a2e-c140d9572669 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5 namespace: kserve-ci-e2e-test resourceVersion: "17671" uid: f9175222-2aa6-4fc1-90f3-9e45fe75afc3 kind: Event lastTimestamp: "2026-04-24T21:46:47Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:47Z" name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5.18a96935be1772af namespace: kserve-ci-e2e-test resourceVersion: "17749" uid: b533c0e6-4ec9-4da9-a7e6-d17e6ddae587 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5 namespace: kserve-ci-e2e-test resourceVersion: "17671" uid: f9175222-2aa6-4fc1-90f3-9e45fe75afc3 kind: Event lastTimestamp: "2026-04-24T21:46:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:47Z" name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5.18a96935c1059b0b namespace: kserve-ci-e2e-test resourceVersion: "17752" uid: e2300d6f-f16e-439f-ba33-762861fbf022 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5 namespace: kserve-ci-e2e-test resourceVersion: "17671" uid: f9175222-2aa6-4fc1-90f3-9e45fe75afc3 kind: Event lastTimestamp: "2026-04-24T21:46:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:47Z" name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5.18a96935c17725ed namespace: kserve-ci-e2e-test resourceVersion: "17755" uid: e683fcea-a346-445b-bf3d-4498bfbec625 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5 namespace: kserve-ci-e2e-test resourceVersion: "17671" uid: f9175222-2aa6-4fc1-90f3-9e45fe75afc3 kind: Event lastTimestamp: "2026-04-24T21:46:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:47Z" name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5.18a96935c18273e3 namespace: kserve-ci-e2e-test resourceVersion: "17756" uid: 5502bce5-021b-461b-bb1c-d0d9ccb353ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5 namespace: kserve-ci-e2e-test resourceVersion: "17671" uid: f9175222-2aa6-4fc1-90f3-9e45fe75afc3 kind: Event lastTimestamp: "2026-04-24T21:46:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:47Z" name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5.18a96935c47df6fb namespace: kserve-ci-e2e-test resourceVersion: "17758" uid: a25b0aa0-6668-4b4e-be14-1e8feb193545 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5 namespace: kserve-ci-e2e-test resourceVersion: "17671" uid: f9175222-2aa6-4fc1-90f3-9e45fe75afc3 kind: Event lastTimestamp: "2026-04-24T21:46:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:47Z" name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5.18a96935c4e9bc2a namespace: kserve-ci-e2e-test resourceVersion: "17759" uid: ff8b2b97-b943-45ab-82e1-d43aac1f7d5c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 15 eventTime: null firstTimestamp: "2026-04-24T21:46:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5 namespace: kserve-ci-e2e-test resourceVersion: "17671" uid: f9175222-2aa6-4fc1-90f3-9e45fe75afc3 kind: Event lastTimestamp: "2026-04-24T21:48:50Z" message: 'Readiness probe failed: dial tcp 10.132.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:50Z" name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5.18a9693635d3b113 namespace: kserve-ci-e2e-test resourceVersion: "18638" uid: 0f0a6ba3-ab8b-49f0-8727-b5985ea235c2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5 namespace: kserve-ci-e2e-test resourceVersion: "17671" uid: f9175222-2aa6-4fc1-90f3-9e45fe75afc3 kind: Event lastTimestamp: "2026-04-24T21:49:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:02Z" name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5.18a9695514d00f00 namespace: kserve-ci-e2e-test resourceVersion: "18761" uid: 9936b99b-4461-4c6f-b34a-4d6e4f13c666 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5 namespace: kserve-ci-e2e-test resourceVersion: "17671" uid: f9175222-2aa6-4fc1-90f3-9e45fe75afc3 kind: Event lastTimestamp: "2026-04-24T21:49:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:02Z" name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5.18a9695514d1138d namespace: kserve-ci-e2e-test resourceVersion: "18764" uid: e8b55d84-5d1e-4672-a336-9d7d22bb739a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5 namespace: kserve-ci-e2e-test resourceVersion: "17671" uid: f9175222-2aa6-4fc1-90f3-9e45fe75afc3 kind: Event lastTimestamp: "2026-04-24T21:49:05Z" message: 'Readiness probe failed: Get "https://10.132.0.41:8643/healthz": dial tcp 10.132.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:05Z" name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5.18a96955e0317164 namespace: kserve-ci-e2e-test resourceVersion: "18825" uid: 834d9fcd-fd91-49ca-a5df-841f041a0ef7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894 namespace: kserve-ci-e2e-test resourceVersion: "17644" uid: cb539ef8-2487-40a5-814c-af046faad1f6 kind: Event lastTimestamp: "2026-04-24T21:46:41Z" message: 'Created pod: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894-fphp5' metadata: creationTimestamp: "2026-04-24T21:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:41Z" name: isvc-sklearn-scale-raw-e8206-predictor-6b9744b894.18a969346f53d0ec namespace: kserve-ci-e2e-test resourceVersion: "17670" uid: d2f3c626-e0ed-4c69-b425-4725615ccd29 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-e8206-predictor namespace: kserve-ci-e2e-test resourceVersion: "17637" uid: 743e0a1d-39cf-41a5-b46f-9a4466d44e56 kind: Event lastTimestamp: "2026-04-24T21:46:41Z" message: Scaled up replica set isvc-sklearn-scale-raw-e8206-predictor-6b9744b894 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:41Z" name: isvc-sklearn-scale-raw-e8206-predictor.18a969346bec1ae2 namespace: kserve-ci-e2e-test resourceVersion: "17648" uid: b4877416-cc53-46a3-bdcb-4d786eddaa24 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:46:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-e8206-predictor namespace: kserve-ci-e2e-test resourceVersion: "17643" uid: 3683a450-d0bc-450f-ae16-459d14f890ab kind: Event lastTimestamp: "2026-04-24T21:47:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:26Z" name: isvc-sklearn-scale-raw-e8206-predictor.18a96937eaebcc07 namespace: kserve-ci-e2e-test resourceVersion: "18051" uid: f43828b4-7f0e-4f7f-8730-e536402df2dc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:46:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-e8206-predictor namespace: kserve-ci-e2e-test resourceVersion: "17643" uid: 3683a450-d0bc-450f-ae16-459d14f890ab kind: Event lastTimestamp: "2026-04-24T21:47:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:26Z" name: isvc-sklearn-scale-raw-e8206-predictor.18a96937eb5b8869 namespace: kserve-ci-e2e-test resourceVersion: "18052" uid: bdfe6d89-216d-463b-bdb4-08c2b41df9ae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:47:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-e8206-predictor namespace: kserve-ci-e2e-test resourceVersion: "17830" uid: 3683a450-d0bc-450f-ae16-459d14f890ab kind: Event lastTimestamp: "2026-04-24T21:48:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:47:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:57Z" name: isvc-sklearn-scale-raw-e8206-predictor.18a9693b6a29b01e namespace: kserve-ci-e2e-test resourceVersion: "18683" uid: 78be12e0-e92b-4f40-a6ea-944167faf08d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:47:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-e8206-predictor namespace: kserve-ci-e2e-test resourceVersion: "17830" uid: 3683a450-d0bc-450f-ae16-459d14f890ab kind: Event lastTimestamp: "2026-04-24T21:48:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:47:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:57Z" name: isvc-sklearn-scale-raw-e8206-predictor.18a9693b6a9aaa44 namespace: kserve-ci-e2e-test resourceVersion: "18684" uid: cef6d256-bb6a-4c07-8953-c72bb9a06432 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T21:46:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-e8206 namespace: kserve-ci-e2e-test resourceVersion: "17628" uid: 7085a449-f6dc-4d9f-b7cb-9a244e3fdec8 kind: Event lastTimestamp: "2026-04-24T21:49:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:01Z" name: isvc-sklearn-scale-raw-e8206.18a969346bc89219 namespace: kserve-ci-e2e-test resourceVersion: "18721" uid: 5bea6bab-3ef1-41c1-a20d-38b81b70e0e1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-e8206 namespace: kserve-ci-e2e-test resourceVersion: "17628" uid: 7085a449-f6dc-4d9f-b7cb-9a244e3fdec8 kind: Event lastTimestamp: "2026-04-24T21:46:41Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-e8206": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-e8206": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:46:41Z" name: isvc-sklearn-scale-raw-e8206.18a969346c74eff3 namespace: kserve-ci-e2e-test resourceVersion: "17654" uid: 5dff1583-76f5-45f1-a2b9-798a86d25fc1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-e8206 namespace: kserve-ci-e2e-test resourceVersion: "18719" uid: 7085a449-f6dc-4d9f-b7cb-9a244e3fdec8 kind: Event lastTimestamp: "2026-04-24T21:49:01Z" message: InferenceService [isvc-sklearn-scale-raw-e8206] is Ready metadata: creationTimestamp: "2026-04-24T21:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:01Z" name: isvc-sklearn-scale-raw-e8206.18a96954d7315fc1 namespace: kserve-ci-e2e-test resourceVersion: "18720" uid: e5758c45-7b08-4994-8f69-ce1f1c290446 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:41:46.132756Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 269a8e34-b07b-4ce4-92b2-f57660f9df87 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx to ip-10-0-134-249.ec2.internal metadata: creationTimestamp: "2026-04-24T21:41:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:41:46Z" name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx.18a968ef922554f5 namespace: kserve-ci-e2e-test resourceVersion: "14872" uid: 3b439bd6-d768-4895-8ca4-df432805ba04 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6688756845-wvlxd source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:41:46Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 269a8e34-b07b-4ce4-92b2-f57660f9df87 kind: Event lastTimestamp: "2026-04-24T21:41:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-62689-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:41:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:46Z" name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx.18a968ef9ca5a33c namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 0288b3e3-d14e-4835-bb34-063a647f1543 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:48Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx namespace: kserve-ci-e2e-test resourceVersion: "14877" uid: 269a8e34-b07b-4ce4-92b2-f57660f9df87 kind: Event lastTimestamp: "2026-04-24T21:41:48Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:41:48Z" name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx.18a968f004852c44 namespace: kserve-ci-e2e-test resourceVersion: "14911" uid: 1a9de49a-1419-463d-8295-15e76705bce0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 269a8e34-b07b-4ce4-92b2-f57660f9df87 kind: Event lastTimestamp: "2026-04-24T21:41:48Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0dd2798f9cdbffc0563bd148612201df0e589ff8c26a6b19d321fd120fc5c097" already present on machine metadata: creationTimestamp: "2026-04-24T21:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:48Z" name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx.18a968f005fcb4a3 namespace: kserve-ci-e2e-test resourceVersion: "14913" uid: 302ca88d-7ccf-42e0-b00c-7147899d5998 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 269a8e34-b07b-4ce4-92b2-f57660f9df87 kind: Event lastTimestamp: "2026-04-24T21:41:48Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:48Z" name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx.18a968f0090ee79a namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: 1381b377-6642-40b0-aac1-ced55782592b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 269a8e34-b07b-4ce4-92b2-f57660f9df87 kind: Event lastTimestamp: "2026-04-24T21:41:48Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:48Z" name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx.18a968f009aa648f namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 9d5b294e-845a-4a64-b573-54ed5c72c44f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 269a8e34-b07b-4ce4-92b2-f57660f9df87 kind: Event lastTimestamp: "2026-04-24T21:41:52Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-24T21:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:52Z" name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx.18a968f10c0c7c8c namespace: kserve-ci-e2e-test resourceVersion: "14968" uid: 4db25a60-76b5-41f0-ba7d-2431df576dca reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:42:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 269a8e34-b07b-4ce4-92b2-f57660f9df87 kind: Event lastTimestamp: "2026-04-24T21:42:10Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 17.581s (17.581s including waiting). Image size: 1306417402 bytes.' metadata: creationTimestamp: "2026-04-24T21:42:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:42:10Z" name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx.18a968f523f92cfb namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: 142bfab7-3098-436a-8eb6-3455672ae026 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:42:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 269a8e34-b07b-4ce4-92b2-f57660f9df87 kind: Event lastTimestamp: "2026-04-24T21:42:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:42:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:42:10Z" name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx.18a968f5272fe7fa namespace: kserve-ci-e2e-test resourceVersion: "15116" uid: 486cf87f-7e05-466d-ba07-c15ed72c45d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:42:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 269a8e34-b07b-4ce4-92b2-f57660f9df87 kind: Event lastTimestamp: "2026-04-24T21:42:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:42:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:42:10Z" name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx.18a968f527a2307e namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: 05f47ac0-6e9c-4349-90ee-ad3b7b059e6e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:42:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 269a8e34-b07b-4ce4-92b2-f57660f9df87 kind: Event lastTimestamp: "2026-04-24T21:42:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:42:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:42:10Z" name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx.18a968f527ad4d84 namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: c20e25bb-6c2c-4995-a15c-b029dd1395c0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:42:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 269a8e34-b07b-4ce4-92b2-f57660f9df87 kind: Event lastTimestamp: "2026-04-24T21:42:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:42:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:42:10Z" name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx.18a968f52ac202bd namespace: kserve-ci-e2e-test resourceVersion: "15119" uid: ab26691f-aa5a-48b1-9910-d3b0ec55ee8b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:42:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 269a8e34-b07b-4ce4-92b2-f57660f9df87 kind: Event lastTimestamp: "2026-04-24T21:42:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:42:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:42:10Z" name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx.18a968f52b31315c namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: f61bd409-90ee-4546-bca7-9efd4b499448 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:42:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 269a8e34-b07b-4ce4-92b2-f57660f9df87 kind: Event lastTimestamp: "2026-04-24T21:43:27Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:42:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:27Z" name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx.18a968f5412fbeb9 namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: 68b1daa0-e8af-48ae-b424-eb7dcd6e17ff reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 269a8e34-b07b-4ce4-92b2-f57660f9df87 kind: Event lastTimestamp: "2026-04-24T21:43:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:26Z" name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx.18a96906ed9d3631 namespace: kserve-ci-e2e-test resourceVersion: "15845" uid: a095c3f9-67fe-4247-855d-35dc9ad7fac9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 269a8e34-b07b-4ce4-92b2-f57660f9df87 kind: Event lastTimestamp: "2026-04-24T21:43:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:26Z" name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx.18a96906ed9d89c6 namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: 63f45bf8-e1bb-468d-986d-b5da4595f46d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 269a8e34-b07b-4ce4-92b2-f57660f9df87 kind: Event lastTimestamp: "2026-04-24T21:43:27Z" message: 'Readiness probe failed: Get "https://10.132.0.36:8643/healthz": dial tcp 10.132.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:27Z" name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx.18a969072f19746d namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: f1f2aea6-34e0-4b27-8c7a-32df284d1684 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69 namespace: kserve-ci-e2e-test resourceVersion: "14852" uid: 89366400-841c-410d-bfc2-cd150b69b279 kind: Event lastTimestamp: "2026-04-24T21:41:46Z" message: 'Created pod: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69-pvkhx' metadata: creationTimestamp: "2026-04-24T21:41:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:46Z" name: isvc-xgboost-graph-raw-62689-predictor-bc95c6d69.18a968ef919432ac namespace: kserve-ci-e2e-test resourceVersion: "14869" uid: 9630f20e-8369-46d6-8521-d8643b2766cb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-62689-predictor namespace: kserve-ci-e2e-test resourceVersion: "14847" uid: 4352f560-c38c-4e9f-a64c-bb345e4c6fe2 kind: Event lastTimestamp: "2026-04-24T21:41:46Z" message: Scaled up replica set isvc-xgboost-graph-raw-62689-predictor-bc95c6d69 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:41:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:46Z" name: isvc-xgboost-graph-raw-62689-predictor.18a968ef8e88ebda namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 72ce8331-1910-4a31-99bf-67e713c7fd2a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:42:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-62689-predictor namespace: kserve-ci-e2e-test resourceVersion: "14851" uid: 9d305a1d-23da-49ed-88e3-47d26a614b9c kind: Event lastTimestamp: "2026-04-24T21:43:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:43:01Z" name: isvc-xgboost-graph-raw-62689-predictor.18a968f30d008f03 namespace: kserve-ci-e2e-test resourceVersion: "15512" uid: d658d109-7485-41a8-b865-24f8816ff949 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:42:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-62689-predictor namespace: kserve-ci-e2e-test resourceVersion: "14851" uid: 9d305a1d-23da-49ed-88e3-47d26a614b9c kind: Event lastTimestamp: "2026-04-24T21:43:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:43:01Z" name: isvc-xgboost-graph-raw-62689-predictor.18a968f30d734f47 namespace: kserve-ci-e2e-test resourceVersion: "15513" uid: d2e5026b-d2ae-4bcb-9b47-f80b1729ec41 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-62689-predictor namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 9d305a1d-23da-49ed-88e3-47d26a614b9c kind: Event lastTimestamp: "2026-04-24T21:43:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:43:16Z" name: isvc-xgboost-graph-raw-62689-predictor.18a96904887e1fc5 namespace: kserve-ci-e2e-test resourceVersion: "15651" uid: 7eecf4a2-8bd4-4e40-9ab6-e8455936fc16 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-62689-predictor namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 9d305a1d-23da-49ed-88e3-47d26a614b9c kind: Event lastTimestamp: "2026-04-24T21:43:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:43:16Z" name: isvc-xgboost-graph-raw-62689-predictor.18a9690488f1dea9 namespace: kserve-ci-e2e-test resourceVersion: "15653" uid: 18e0a814-336c-47f6-85a9-7d43a54f055e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-24T21:41:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-62689 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: a44e6acd-7fb9-4b6e-9737-866816a26c12 kind: Event lastTimestamp: "2026-04-24T21:43:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:41:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:43:07Z" name: isvc-xgboost-graph-raw-62689.18a968ef8e572389 namespace: kserve-ci-e2e-test resourceVersion: "15564" uid: 4d667488-0512-4837-83bb-9ecab15151a9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-62689 namespace: kserve-ci-e2e-test resourceVersion: "15562" uid: a44e6acd-7fb9-4b6e-9737-866816a26c12 kind: Event lastTimestamp: "2026-04-24T21:43:07Z" message: InferenceService [isvc-xgboost-graph-raw-62689] is Ready metadata: creationTimestamp: "2026-04-24T21:43:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:43:07Z" name: isvc-xgboost-graph-raw-62689.18a969028cb320ea namespace: kserve-ci-e2e-test resourceVersion: "15563" uid: 5f82b651-17ee-43fb-9329-03af5b0a2b05 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:43:26.637649Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz namespace: kserve-ci-e2e-test resourceVersion: "15856" uid: 4560568f-042c-47ee-838e-0ade81f72494 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz to ip-10-0-134-249.ec2.internal metadata: creationTimestamp: "2026-04-24T21:43:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:43:26Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz.18a96906f8b45a14 namespace: kserve-ci-e2e-test resourceVersion: "15861" uid: 908fb3d0-2579-48ba-b20b-3d5ab31366fd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6688756845-wvlxd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: 4560568f-042c-47ee-838e-0ade81f72494 kind: Event lastTimestamp: "2026-04-24T21:43:27Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:43:27Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz.18a9690711b58eca namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: 965d4011-d3de-41c2-bf75-59611c377cc3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: 4560568f-042c-47ee-838e-0ade81f72494 kind: Event lastTimestamp: "2026-04-24T21:43:27Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0dd2798f9cdbffc0563bd148612201df0e589ff8c26a6b19d321fd120fc5c097" already present on machine metadata: creationTimestamp: "2026-04-24T21:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:27Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz.18a9690713686f0d namespace: kserve-ci-e2e-test resourceVersion: "15875" uid: ad0f8b20-63d2-41ad-b587-d5a5b32a5543 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: 4560568f-042c-47ee-838e-0ade81f72494 kind: Event lastTimestamp: "2026-04-24T21:43:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:27Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz.18a9690716e5a401 namespace: kserve-ci-e2e-test resourceVersion: "15879" uid: c8cf6b92-c86d-48e0-842f-ac5ab84b0d5d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: 4560568f-042c-47ee-838e-0ade81f72494 kind: Event lastTimestamp: "2026-04-24T21:43:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:27Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz.18a9690717564e19 namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: 5bb5f38c-9318-43bc-88e5-7fd70e5747e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: 4560568f-042c-47ee-838e-0ade81f72494 kind: Event lastTimestamp: "2026-04-24T21:43:30Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-24T21:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:30Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz.18a96907f2bdca9e namespace: kserve-ci-e2e-test resourceVersion: "15933" uid: f88c153f-63dc-4b31-865c-601b932708b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: 4560568f-042c-47ee-838e-0ade81f72494 kind: Event lastTimestamp: "2026-04-24T21:43:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:30Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz.18a96907f7c18c67 namespace: kserve-ci-e2e-test resourceVersion: "15937" uid: 416a03fe-0c20-44c5-b0f9-15a72488d85c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: 4560568f-042c-47ee-838e-0ade81f72494 kind: Event lastTimestamp: "2026-04-24T21:43:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:30Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz.18a96907f865f1a8 namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: e86eea52-0aa6-43f6-8016-a6916af569ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: 4560568f-042c-47ee-838e-0ade81f72494 kind: Event lastTimestamp: "2026-04-24T21:43:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:30Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz.18a96907f8731f93 namespace: kserve-ci-e2e-test resourceVersion: "15941" uid: 014d0b53-2eec-4887-be6a-5fc72b7d280b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: 4560568f-042c-47ee-838e-0ade81f72494 kind: Event lastTimestamp: "2026-04-24T21:43:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:30Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz.18a96907fb9ee439 namespace: kserve-ci-e2e-test resourceVersion: "15944" uid: 51563020-c579-4b3c-9b73-a3071fb83202 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: 4560568f-042c-47ee-838e-0ade81f72494 kind: Event lastTimestamp: "2026-04-24T21:43:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:30Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz.18a96907fc38ee6b namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: 414f1b41-5f44-4aae-82a8-5f35f2ad8264 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T21:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: 4560568f-042c-47ee-838e-0ade81f72494 kind: Event lastTimestamp: "2026-04-24T21:45:08Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:08Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz.18a969086ad7259e namespace: kserve-ci-e2e-test resourceVersion: "16845" uid: a85a6615-7e5e-452b-aeba-a8cfe1d243db reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: 4560568f-042c-47ee-838e-0ade81f72494 kind: Event lastTimestamp: "2026-04-24T21:45:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:45:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:06Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz.18a9691e4415b2c0 namespace: kserve-ci-e2e-test resourceVersion: "16811" uid: ddebffc8-5860-4a24-9492-9ccd486439d7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: 4560568f-042c-47ee-838e-0ade81f72494 kind: Event lastTimestamp: "2026-04-24T21:45:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:45:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:06Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz.18a9691e4416c97b namespace: kserve-ci-e2e-test resourceVersion: "16812" uid: 2953277e-5146-4c83-b177-aa7b3889876e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: 4560568f-042c-47ee-838e-0ade81f72494 kind: Event lastTimestamp: "2026-04-24T21:45:08Z" message: 'Readiness probe failed: Get "https://10.132.0.38:8643/healthz": dial tcp 10.132.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:45:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:08Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz.18a9691ec50e405d namespace: kserve-ci-e2e-test resourceVersion: "16843" uid: b4a6ec03-7aac-443f-aded-ea6042dec24d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765 namespace: kserve-ci-e2e-test resourceVersion: "15821" uid: 25c23500-7222-49b3-a3b4-499547686256 kind: Event lastTimestamp: "2026-04-24T21:43:26Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765-fcrsz' metadata: creationTimestamp: "2026-04-24T21:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:43:26Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765.18a96906f810cf7f namespace: kserve-ci-e2e-test resourceVersion: "15858" uid: a942eb27-5e17-4645-b8ce-b3937a9a58a0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15816" uid: d564c876-11dd-46b6-8495-5a9b6307af1a kind: Event lastTimestamp: "2026-04-24T21:43:26Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-2b4f4-predictor-7b7f6f6765 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:43:26Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor.18a96906e89b18d3 namespace: kserve-ci-e2e-test resourceVersion: "15824" uid: 36949f8d-710b-4181-b2a6-cb35f3367369 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:43:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: 78322c22-882b-4913-949d-b3bd13ccca95 kind: Event lastTimestamp: "2026-04-24T21:44:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:43:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:44:11Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor.18a9690a67289812 namespace: kserve-ci-e2e-test resourceVersion: "16272" uid: 9d6c50f9-5b04-4cf4-be2a-ee43d2bb3b15 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:43:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: 78322c22-882b-4913-949d-b3bd13ccca95 kind: Event lastTimestamp: "2026-04-24T21:44:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:43:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:44:11Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor.18a9690a67a2d846 namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: d212b756-e5e8-45e6-b50c-7295c5f77769 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:43:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: 78322c22-882b-4913-949d-b3bd13ccca95 kind: Event lastTimestamp: "2026-04-24T21:44:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:44:56Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor.18a9690de67e86c3 namespace: kserve-ci-e2e-test resourceVersion: "16631" uid: 8eb462e1-8082-4dd9-ac17-76e36e7af089 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:43:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: 78322c22-882b-4913-949d-b3bd13ccca95 kind: Event lastTimestamp: "2026-04-24T21:44:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:44:56Z" name: isvc-xgboost-graph-raw-hpa-2b4f4-predictor.18a9690de6faed3f namespace: kserve-ci-e2e-test resourceVersion: "16635" uid: 2db5abc1-f559-4cf9-8fc3-549524df1ff1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:43:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-2b4f4 namespace: kserve-ci-e2e-test resourceVersion: "15799" uid: 70076344-84ad-43ab-8e61-d1934dff4568 kind: Event lastTimestamp: "2026-04-24T21:44:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:44:29Z" name: isvc-xgboost-graph-raw-hpa-2b4f4.18a96906e872e97f namespace: kserve-ci-e2e-test resourceVersion: "16406" uid: d2d8d21c-73b0-4166-9623-f3f880b1177a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-2b4f4 namespace: kserve-ci-e2e-test resourceVersion: "15799" uid: 70076344-84ad-43ab-8e61-d1934dff4568 kind: Event lastTimestamp: "2026-04-24T21:43:26Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-2b4f4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-2b4f4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:43:26Z" name: isvc-xgboost-graph-raw-hpa-2b4f4.18a96906e9037ae2 namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: 9e65d866-5e8e-4416-b200-42ed59c8ddbc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:44:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-2b4f4 namespace: kserve-ci-e2e-test resourceVersion: "16403" uid: 70076344-84ad-43ab-8e61-d1934dff4568 kind: Event lastTimestamp: "2026-04-24T21:44:28Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-2b4f4] is Ready metadata: creationTimestamp: "2026-04-24T21:44:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:44:28Z" name: isvc-xgboost-graph-raw-hpa-2b4f4.18a969157a6959d0 namespace: kserve-ci-e2e-test resourceVersion: "16405" uid: 748fc4a8-8f6d-4976-9061-2ec796282597 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:45:06.637247Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn namespace: kserve-ci-e2e-test resourceVersion: "16785" uid: 1cfcc287-0d5d-4ba7-be93-a76c1ad74d7b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn to ip-10-0-134-249.ec2.internal metadata: creationTimestamp: "2026-04-24T21:45:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:45:06Z" name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn.18a9691e412513d7 namespace: kserve-ci-e2e-test resourceVersion: "16790" uid: 537c050a-293f-4862-9ba3-31de652bb09f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6688756845-wvlxd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:07Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn namespace: kserve-ci-e2e-test resourceVersion: "16803" uid: 1cfcc287-0d5d-4ba7-be93-a76c1ad74d7b kind: Event lastTimestamp: "2026-04-24T21:45:07Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:45:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:45:07Z" name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn.18a9691e5a19d835 namespace: kserve-ci-e2e-test resourceVersion: "16819" uid: b3adff81-c936-4e5b-9684-9eb63a33f4e9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: 1cfcc287-0d5d-4ba7-be93-a76c1ad74d7b kind: Event lastTimestamp: "2026-04-24T21:45:07Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-24T21:45:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:07Z" name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn.18a9691e5b84fea0 namespace: kserve-ci-e2e-test resourceVersion: "16821" uid: 7c1ce4e1-e991-49af-8c99-ae6471dd3ea5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: 1cfcc287-0d5d-4ba7-be93-a76c1ad74d7b kind: Event lastTimestamp: "2026-04-24T21:45:08Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.024s (1.024s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-24T21:45:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:08Z" name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn.18a9691e988ea4df namespace: kserve-ci-e2e-test resourceVersion: "16829" uid: bccb8384-48f7-4557-b4bf-67baca05fe38 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: 1cfcc287-0d5d-4ba7-be93-a76c1ad74d7b kind: Event lastTimestamp: "2026-04-24T21:45:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:45:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:08Z" name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn.18a9691e9bbe0906 namespace: kserve-ci-e2e-test resourceVersion: "16830" uid: 0fc00116-1eed-43f6-99d5-06ed5678b361 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: 1cfcc287-0d5d-4ba7-be93-a76c1ad74d7b kind: Event lastTimestamp: "2026-04-24T21:45:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:45:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:08Z" name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn.18a9691e9c32ec0e namespace: kserve-ci-e2e-test resourceVersion: "16831" uid: 463c1985-86a4-48aa-a9c7-148caf8c2d5c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: 1cfcc287-0d5d-4ba7-be93-a76c1ad74d7b kind: Event lastTimestamp: "2026-04-24T21:45:08Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:45:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:08Z" name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn.18a9691e9c3ee415 namespace: kserve-ci-e2e-test resourceVersion: "16832" uid: 66a3e20e-e68e-4ec4-8a0a-52f655a0f4fc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: 1cfcc287-0d5d-4ba7-be93-a76c1ad74d7b kind: Event lastTimestamp: "2026-04-24T21:45:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:45:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:08Z" name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn.18a9691e9f2fbaca namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: d06b0eec-dc11-413b-83b7-4bb8bcce374c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: 1cfcc287-0d5d-4ba7-be93-a76c1ad74d7b kind: Event lastTimestamp: "2026-04-24T21:45:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:45:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:08Z" name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn.18a9691e9f9eb6e1 namespace: kserve-ci-e2e-test resourceVersion: "16834" uid: 482f53d0-c821-440a-87b7-9111d28674fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: 1cfcc287-0d5d-4ba7-be93-a76c1ad74d7b kind: Event lastTimestamp: "2026-04-24T21:46:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:41Z" name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn.18a9693474729fb8 namespace: kserve-ci-e2e-test resourceVersion: "17695" uid: 9faf4b7b-8da5-4d55-92f4-9f3c3f03772e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: 1cfcc287-0d5d-4ba7-be93-a76c1ad74d7b kind: Event lastTimestamp: "2026-04-24T21:46:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:41Z" name: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn.18a9693474731431 namespace: kserve-ci-e2e-test resourceVersion: "17696" uid: 0e630903-691d-48d7-b857-39c4e111dc8a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-fb0e6-predictor-774f648d7f namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: f48ac664-0d1e-4e11-8afd-9853980715d3 kind: Event lastTimestamp: "2026-04-24T21:45:06Z" message: 'Created pod: message-dumper-raw-fb0e6-predictor-774f648d7f-fgmbn' metadata: creationTimestamp: "2026-04-24T21:45:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:06Z" name: message-dumper-raw-fb0e6-predictor-774f648d7f.18a9691e40bb643e namespace: kserve-ci-e2e-test resourceVersion: "16788" uid: 18471f66-32ae-4162-9561-35f170b673a7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-fb0e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "16762" uid: c2f5b4e5-e840-4a3a-a797-25ac11025aaf kind: Event lastTimestamp: "2026-04-24T21:45:06Z" message: Scaled up replica set message-dumper-raw-fb0e6-predictor-774f648d7f from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:45:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:06Z" name: message-dumper-raw-fb0e6-predictor.18a9691e3d5d62cb namespace: kserve-ci-e2e-test resourceVersion: "16774" uid: 2cc8c150-da76-4b20-99eb-1f8e798ac6dd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:45:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-fb0e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "16769" uid: 3815e690-9ac4-42f7-b754-44e959ed2a0f kind: Event lastTimestamp: "2026-04-24T21:45:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:36Z" name: message-dumper-raw-fb0e6-predictor.18a96921bc74fdb0 namespace: kserve-ci-e2e-test resourceVersion: "17138" uid: 95988f80-fbbf-4f91-b870-6a486f7d5880 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:45:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-fb0e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "16769" uid: 3815e690-9ac4-42f7-b754-44e959ed2a0f kind: Event lastTimestamp: "2026-04-24T21:45:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:36Z" name: message-dumper-raw-fb0e6-predictor.18a96921bce1a37c namespace: kserve-ci-e2e-test resourceVersion: "17139" uid: 6aea9b2e-f58c-4b36-ac70-43eb0d6c75dc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:45:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-fb0e6 namespace: kserve-ci-e2e-test resourceVersion: "16746" uid: b0a6f0cc-9d90-4c58-bd54-3024aabae84e kind: Event lastTimestamp: "2026-04-24T21:45:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:45:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:16Z" name: message-dumper-raw-fb0e6.18a9691e3d2d5a38 namespace: kserve-ci-e2e-test resourceVersion: "16913" uid: 74c81a52-f708-489b-b271-008e8a000d5e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-fb0e6 namespace: kserve-ci-e2e-test resourceVersion: "16746" uid: b0a6f0cc-9d90-4c58-bd54-3024aabae84e kind: Event lastTimestamp: "2026-04-24T21:45:06Z" message: 'Failed to update status for InferenceService "message-dumper-raw-fb0e6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-fb0e6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:45:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:06Z" name: message-dumper-raw-fb0e6.18a9691e3db3c82b namespace: kserve-ci-e2e-test resourceVersion: "16776" uid: 5b53d931-ae01-4422-8779-c8961a38f63a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-fb0e6 namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: b0a6f0cc-9d90-4c58-bd54-3024aabae84e kind: Event lastTimestamp: "2026-04-24T21:45:16Z" message: InferenceService [message-dumper-raw-fb0e6] is Ready metadata: creationTimestamp: "2026-04-24T21:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:16Z" name: message-dumper-raw-fb0e6.18a969207edddfe7 namespace: kserve-ci-e2e-test resourceVersion: "16912" uid: 5ff8d58b-3ec7-4412-8f02-0f9809bc20f3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:43:16.102967Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-62689-7d88bd74c9-ql9vk namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: 82ed1038-5d08-47ce-a138-1034c8384dec kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-62689-7d88bd74c9-ql9vk to ip-10-0-135-27.ec2.internal metadata: creationTimestamp: "2026-04-24T21:43:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:43:16Z" name: model-chainer-raw-62689-7d88bd74c9-ql9vk.18a9690484c9c519 namespace: kserve-ci-e2e-test resourceVersion: "15643" uid: 52017078-5aed-46f8-ba09-ed65d5167c28 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6688756845-wvlxd source: {} type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:43:16Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-62689-7d88bd74c9-ql9vk namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: 82ed1038-5d08-47ce-a138-1034c8384dec kind: Event lastTimestamp: "2026-04-24T21:43:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-62689-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:41Z" name: model-chainer-raw-62689-7d88bd74c9-ql9vk.18a96904944f80f2 namespace: kserve-ci-e2e-test resourceVersion: "16052" uid: e7118ccb-1f5c-4954-95c9-608ef6f98e58 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-27.ec2.internal source: component: kubelet host: ip-10-0-135-27.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:17Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-62689-7d88bd74c9-ql9vk namespace: kserve-ci-e2e-test resourceVersion: "15646" uid: 82ed1038-5d08-47ce-a138-1034c8384dec kind: Event lastTimestamp: "2026-04-24T21:43:17Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:43:17Z" name: model-chainer-raw-62689-7d88bd74c9-ql9vk.18a96904c1999ac4 namespace: kserve-ci-e2e-test resourceVersion: "15669" uid: cdaa413a-beb6-4c1c-83cb-1f59d1ea142d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-62689} kind: Pod name: model-chainer-raw-62689-7d88bd74c9-ql9vk namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: 82ed1038-5d08-47ce-a138-1034c8384dec kind: Event lastTimestamp: "2026-04-24T21:43:17Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:bfb5b2782772ad46be0a5c41e64aded1dd42f09d6e2e6f1f1c3194223df058cb" metadata: creationTimestamp: "2026-04-24T21:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:17Z" name: model-chainer-raw-62689-7d88bd74c9-ql9vk.18a96904c31443da namespace: kserve-ci-e2e-test resourceVersion: "15672" uid: 1ed1817e-55af-4a47-87e9-d5cbe1879037 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-27.ec2.internal source: component: kubelet host: ip-10-0-135-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-62689} kind: Pod name: model-chainer-raw-62689-7d88bd74c9-ql9vk namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: 82ed1038-5d08-47ce-a138-1034c8384dec kind: Event lastTimestamp: "2026-04-24T21:43:19Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:bfb5b2782772ad46be0a5c41e64aded1dd42f09d6e2e6f1f1c3194223df058cb" in 2.094s (2.094s including waiting). Image size: 216346202 bytes.' metadata: creationTimestamp: "2026-04-24T21:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:19Z" name: model-chainer-raw-62689-7d88bd74c9-ql9vk.18a969053fe6bd0b namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: 27ba6470-406d-4fbe-b340-c08ca5db0e3f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-27.ec2.internal source: component: kubelet host: ip-10-0-135-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-62689} kind: Pod name: model-chainer-raw-62689-7d88bd74c9-ql9vk namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: 82ed1038-5d08-47ce-a138-1034c8384dec kind: Event lastTimestamp: "2026-04-24T21:43:19Z" message: 'Created container: model-chainer-raw-62689' metadata: creationTimestamp: "2026-04-24T21:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:19Z" name: model-chainer-raw-62689-7d88bd74c9-ql9vk.18a9690542bbaedc namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: 05a8cfce-84ef-42db-87b8-746dfd9080e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-27.ec2.internal source: component: kubelet host: ip-10-0-135-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-62689} kind: Pod name: model-chainer-raw-62689-7d88bd74c9-ql9vk namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: 82ed1038-5d08-47ce-a138-1034c8384dec kind: Event lastTimestamp: "2026-04-24T21:43:19Z" message: Started container model-chainer-raw-62689 metadata: creationTimestamp: "2026-04-24T21:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:19Z" name: model-chainer-raw-62689-7d88bd74c9-ql9vk.18a969054328b32c namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: 4759b00d-7a9b-4c90-a559-300c7c03a927 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-27.ec2.internal source: component: kubelet host: ip-10-0-135-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-62689} kind: Pod name: model-chainer-raw-62689-7d88bd74c9-ql9vk namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: 82ed1038-5d08-47ce-a138-1034c8384dec kind: Event lastTimestamp: "2026-04-24T21:43:26Z" message: Stopping container model-chainer-raw-62689 metadata: creationTimestamp: "2026-04-24T21:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:26Z" name: model-chainer-raw-62689-7d88bd74c9-ql9vk.18a96906db231601 namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: c9ed6434-3a5f-4226-805c-c61d25bab332 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-27.ec2.internal source: component: kubelet host: ip-10-0-135-27.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:43:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-62689} kind: Pod name: model-chainer-raw-62689-7d88bd74c9-ql9vk namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: 82ed1038-5d08-47ce-a138-1034c8384dec kind: Event lastTimestamp: "2026-04-24T21:43:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:55Z" name: model-chainer-raw-62689-7d88bd74c9-ql9vk.18a96907fb30dae0 namespace: kserve-ci-e2e-test resourceVersion: "16147" uid: 14883651-3910-423f-a719-3cb5d6dff678 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-27.ec2.internal source: component: kubelet host: ip-10-0-135-27.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-62689-7d88bd74c9 namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: c907d9f4-6309-4939-a831-c7a2b3b17b45 kind: Event lastTimestamp: "2026-04-24T21:43:16Z" message: 'Created pod: model-chainer-raw-62689-7d88bd74c9-ql9vk' metadata: creationTimestamp: "2026-04-24T21:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:43:16Z" name: model-chainer-raw-62689-7d88bd74c9.18a96904841e65d0 namespace: kserve-ci-e2e-test resourceVersion: "15640" uid: e2679fd6-0a3f-424f-ac1a-03e6e3fa62a9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-62689 namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: a147436f-6214-4cb1-a30a-6bfc3ba15bca kind: Event lastTimestamp: "2026-04-24T21:43:16Z" message: Scaled up replica set model-chainer-raw-62689-7d88bd74c9 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:43:16Z" name: model-chainer-raw-62689.18a96904831196a8 namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: 252ce007-f9b5-429b-828f-3a856d96a48a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-62689 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: 4d2d6d97-ea16-4722-8fc2-bf7db145ae46 kind: Event lastTimestamp: "2026-04-24T21:43:16Z" message: InferenceGraph [model-chainer-raw-62689] is Ready metadata: creationTimestamp: "2026-04-24T21:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:43:16Z" name: model-chainer-raw-62689.18a96904896d76ba namespace: kserve-ci-e2e-test resourceVersion: "15656" uid: a17f09a0-1a5e-4943-b177-f7d6bed89481 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:44:56.382571Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-2b4f4-5cbd8c8895-24wlt namespace: kserve-ci-e2e-test resourceVersion: "16616" uid: de08f8b7-f188-4003-9a25-93f7161c3fc7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-2b4f4-5cbd8c8895-24wlt to ip-10-0-135-27.ec2.internal metadata: creationTimestamp: "2026-04-24T21:44:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:44:56Z" name: model-chainer-raw-hpa-2b4f4-5cbd8c8895-24wlt.18a9691bddeb2d01 namespace: kserve-ci-e2e-test resourceVersion: "16622" uid: dfc3498b-7ae0-4efb-9c77-9b892da9437f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6688756845-wvlxd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:44:56Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-2b4f4-5cbd8c8895-24wlt namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: de08f8b7-f188-4003-9a25-93f7161c3fc7 kind: Event lastTimestamp: "2026-04-24T21:44:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-2b4f4-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:44:56Z" name: model-chainer-raw-hpa-2b4f4-5cbd8c8895-24wlt.18a9691beaa4fac4 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: b8cccfa3-e4a2-4d08-8666-2e969f622686 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-27.ec2.internal source: component: kubelet host: ip-10-0-135-27.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:44:57Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-2b4f4-5cbd8c8895-24wlt namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: de08f8b7-f188-4003-9a25-93f7161c3fc7 kind: Event lastTimestamp: "2026-04-24T21:44:57Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:44:57Z" name: model-chainer-raw-hpa-2b4f4-5cbd8c8895-24wlt.18a9691c1a86c685 namespace: kserve-ci-e2e-test resourceVersion: "16652" uid: 33e15e67-b933-4b4c-9024-587547048069 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:44:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-2b4f4} kind: Pod name: model-chainer-raw-hpa-2b4f4-5cbd8c8895-24wlt namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: de08f8b7-f188-4003-9a25-93f7161c3fc7 kind: Event lastTimestamp: "2026-04-24T21:44:57Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bfb5b2782772ad46be0a5c41e64aded1dd42f09d6e2e6f1f1c3194223df058cb" already present on machine metadata: creationTimestamp: "2026-04-24T21:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:44:57Z" name: model-chainer-raw-hpa-2b4f4-5cbd8c8895-24wlt.18a9691c1bed8f7a namespace: kserve-ci-e2e-test resourceVersion: "16654" uid: 46e116ef-8655-4c70-8539-f10af228c1af reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-27.ec2.internal source: component: kubelet host: ip-10-0-135-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:44:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-2b4f4} kind: Pod name: model-chainer-raw-hpa-2b4f4-5cbd8c8895-24wlt namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: de08f8b7-f188-4003-9a25-93f7161c3fc7 kind: Event lastTimestamp: "2026-04-24T21:44:57Z" message: 'Created container: model-chainer-raw-hpa-2b4f4' metadata: creationTimestamp: "2026-04-24T21:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:44:57Z" name: model-chainer-raw-hpa-2b4f4-5cbd8c8895-24wlt.18a9691c1e7fda8d namespace: kserve-ci-e2e-test resourceVersion: "16655" uid: 750ce02b-5611-47bb-9243-422fb62e932d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-27.ec2.internal source: component: kubelet host: ip-10-0-135-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:44:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-2b4f4} kind: Pod name: model-chainer-raw-hpa-2b4f4-5cbd8c8895-24wlt namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: de08f8b7-f188-4003-9a25-93f7161c3fc7 kind: Event lastTimestamp: "2026-04-24T21:44:57Z" message: Started container model-chainer-raw-hpa-2b4f4 metadata: creationTimestamp: "2026-04-24T21:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:44:57Z" name: model-chainer-raw-hpa-2b4f4-5cbd8c8895-24wlt.18a9691c1eed155d namespace: kserve-ci-e2e-test resourceVersion: "16656" uid: eb61ae4e-38bb-4d3d-8c66-79db9e59606e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-27.ec2.internal source: component: kubelet host: ip-10-0-135-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-2b4f4} kind: Pod name: model-chainer-raw-hpa-2b4f4-5cbd8c8895-24wlt namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: de08f8b7-f188-4003-9a25-93f7161c3fc7 kind: Event lastTimestamp: "2026-04-24T21:45:06Z" message: Stopping container model-chainer-raw-hpa-2b4f4 metadata: creationTimestamp: "2026-04-24T21:45:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:06Z" name: model-chainer-raw-hpa-2b4f4-5cbd8c8895-24wlt.18a9691e357463be namespace: kserve-ci-e2e-test resourceVersion: "16743" uid: f91c2615-f643-4ff1-b93b-ab44338107c9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-27.ec2.internal source: component: kubelet host: ip-10-0-135-27.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:45:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-2b4f4} kind: Pod name: model-chainer-raw-hpa-2b4f4-5cbd8c8895-24wlt namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: de08f8b7-f188-4003-9a25-93f7161c3fc7 kind: Event lastTimestamp: "2026-04-24T21:45:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:45:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:34Z" name: model-chainer-raw-hpa-2b4f4-5cbd8c8895-24wlt.18a9691edc8a0ec1 namespace: kserve-ci-e2e-test resourceVersion: "17120" uid: ecec2a0d-7644-4b9b-b6b4-2ec645696b15 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-27.ec2.internal source: component: kubelet host: ip-10-0-135-27.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:44:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-2b4f4-5cbd8c8895 namespace: kserve-ci-e2e-test resourceVersion: "16613" uid: e1831706-74a5-4ecc-83b7-0984af445c1a kind: Event lastTimestamp: "2026-04-24T21:44:56Z" message: 'Created pod: model-chainer-raw-hpa-2b4f4-5cbd8c8895-24wlt' metadata: creationTimestamp: "2026-04-24T21:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:44:56Z" name: model-chainer-raw-hpa-2b4f4-5cbd8c8895.18a9691bdd7944c2 namespace: kserve-ci-e2e-test resourceVersion: "16617" uid: ac156ed8-9c1b-4947-ae64-b690061a37d9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:44:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-2b4f4 namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: ec374f61-a66a-4ce9-b52c-a56535ced68e kind: Event lastTimestamp: "2026-04-24T21:44:56Z" message: Scaled up replica set model-chainer-raw-hpa-2b4f4-5cbd8c8895 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:44:56Z" name: model-chainer-raw-hpa-2b4f4.18a9691bdc9c0e90 namespace: kserve-ci-e2e-test resourceVersion: "16614" uid: 478fb78d-7480-4928-b45b-824b22af1f1d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:44:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-2b4f4 namespace: kserve-ci-e2e-test resourceVersion: "16630" uid: db387057-c75a-4c08-8516-17d9b0ed03d1 kind: Event lastTimestamp: "2026-04-24T21:44:56Z" message: InferenceGraph [model-chainer-raw-hpa-2b4f4] is Ready metadata: creationTimestamp: "2026-04-24T21:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:44:56Z" name: model-chainer-raw-hpa-2b4f4.18a9691be38e34a3 namespace: kserve-ci-e2e-test resourceVersion: "16634" uid: ab6f5cd1-a372-4626-8056-7f11b35c283e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:50:50.645826Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: b033b505-1678-4e29-bb5b-ae293c77330c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-1ac54-predictor-995c89dc6-45xgf to ip-10-0-134-249.ec2.internal metadata: creationTimestamp: "2026-04-24T21:50:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:50:50Z" name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf.18a9696e59a767ce namespace: kserve-ci-e2e-test resourceVersion: "19807" uid: 2d282ab7-2943-43c2-85ed-be151ae6d517 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6688756845-wvlxd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:51Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: b033b505-1678-4e29-bb5b-ae293c77330c kind: Event lastTimestamp: "2026-04-24T21:50:51Z" message: Add eth0 [10.132.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:50:51Z" name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf.18a9696e7f006587 namespace: kserve-ci-e2e-test resourceVersion: "19824" uid: 48fd29ac-9af3-4eb5-bdde-9d3f6d9e1c74 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: b033b505-1678-4e29-bb5b-ae293c77330c kind: Event lastTimestamp: "2026-04-24T21:50:51Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0dd2798f9cdbffc0563bd148612201df0e589ff8c26a6b19d321fd120fc5c097" already present on machine metadata: creationTimestamp: "2026-04-24T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:51Z" name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf.18a9696e804798f6 namespace: kserve-ci-e2e-test resourceVersion: "19826" uid: 4b2e960a-d626-4861-948c-76b0d322d390 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: b033b505-1678-4e29-bb5b-ae293c77330c kind: Event lastTimestamp: "2026-04-24T21:50:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:51Z" name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf.18a9696e8302e124 namespace: kserve-ci-e2e-test resourceVersion: "19827" uid: 0913f3c8-ecaf-4ef8-9320-6c6e0f710111 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: b033b505-1678-4e29-bb5b-ae293c77330c kind: Event lastTimestamp: "2026-04-24T21:50:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:51Z" name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf.18a9696e837062d4 namespace: kserve-ci-e2e-test resourceVersion: "19828" uid: 046ac239-95c8-49a9-a038-ba070d5b5b0d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: b033b505-1678-4e29-bb5b-ae293c77330c kind: Event lastTimestamp: "2026-04-24T21:50:55Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:55Z" name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf.18a9696f73b5858a namespace: kserve-ci-e2e-test resourceVersion: "19868" uid: c6daea9c-8846-4ef7-8856-76189da75627 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: b033b505-1678-4e29-bb5b-ae293c77330c kind: Event lastTimestamp: "2026-04-24T21:50:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:55Z" name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf.18a9696f7693afaa namespace: kserve-ci-e2e-test resourceVersion: "19870" uid: b55f1ab0-0c31-4a4a-ad57-ab8f7201ec76 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: b033b505-1678-4e29-bb5b-ae293c77330c kind: Event lastTimestamp: "2026-04-24T21:50:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:55Z" name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf.18a9696f76ff06c6 namespace: kserve-ci-e2e-test resourceVersion: "19871" uid: 63da9f88-390a-432e-a9ed-148ccfe43a90 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: b033b505-1678-4e29-bb5b-ae293c77330c kind: Event lastTimestamp: "2026-04-24T21:50:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:55Z" name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf.18a9696f770b5ca9 namespace: kserve-ci-e2e-test resourceVersion: "19874" uid: 22ef240d-1a61-4921-8b4e-8a8344dab076 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: b033b505-1678-4e29-bb5b-ae293c77330c kind: Event lastTimestamp: "2026-04-24T21:50:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:55Z" name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf.18a9696f79bd217c namespace: kserve-ci-e2e-test resourceVersion: "19875" uid: a0dbcb2b-64a3-4a93-aa9f-0c3a8da99f71 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: b033b505-1678-4e29-bb5b-ae293c77330c kind: Event lastTimestamp: "2026-04-24T21:50:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:55Z" name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf.18a9696f7a276aab namespace: kserve-ci-e2e-test resourceVersion: "19876" uid: 560d46e5-4558-44ac-aa19-c311ad5888c8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:50:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: b033b505-1678-4e29-bb5b-ae293c77330c kind: Event lastTimestamp: "2026-04-24T21:52:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:52:13Z" name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf.18a9696feb6c584a namespace: kserve-ci-e2e-test resourceVersion: "20509" uid: 4c51d08d-5929-4d74-8665-7a960934b97c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:52:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: b033b505-1678-4e29-bb5b-ae293c77330c kind: Event lastTimestamp: "2026-04-24T21:52:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:52:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:52:10Z" name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf.18a9698102e6e5b9 namespace: kserve-ci-e2e-test resourceVersion: "20440" uid: cc5ba551-b96c-4d0d-ab5e-6c4321ff65af reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:52:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: b033b505-1678-4e29-bb5b-ae293c77330c kind: Event lastTimestamp: "2026-04-24T21:52:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:52:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:52:10Z" name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf.18a9698102e77834 namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: b28a8af4-9d76-4cb9-9d8c-7c97ed107514 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:52:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: b033b505-1678-4e29-bb5b-ae293c77330c kind: Event lastTimestamp: "2026-04-24T21:52:13Z" message: 'Readiness probe failed: Get "https://10.132.0.45:8643/healthz": dial tcp 10.132.0.45:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:52:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:52:13Z" name: raw-sklearn-1ac54-predictor-995c89dc6-45xgf.18a969819d836cfc namespace: kserve-ci-e2e-test resourceVersion: "20508" uid: a759c34d-a82d-4a62-9d60-27452d907bae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-1ac54-predictor-995c89dc6 namespace: kserve-ci-e2e-test resourceVersion: "19793" uid: 39975124-e06a-48b6-b3b9-ad51a1fe9b83 kind: Event lastTimestamp: "2026-04-24T21:50:50Z" message: 'Created pod: raw-sklearn-1ac54-predictor-995c89dc6-45xgf' metadata: creationTimestamp: "2026-04-24T21:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:50Z" name: raw-sklearn-1ac54-predictor-995c89dc6.18a9696e5942b570 namespace: kserve-ci-e2e-test resourceVersion: "19804" uid: bfb37c07-1f6c-4197-b27a-9fb6e8a02b4b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-1ac54-predictor namespace: kserve-ci-e2e-test resourceVersion: "19791" uid: 97baf1fe-425c-47e4-8b58-a11da9dfeb0e kind: Event lastTimestamp: "2026-04-24T21:50:50Z" message: Scaled up replica set raw-sklearn-1ac54-predictor-995c89dc6 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:50Z" name: raw-sklearn-1ac54-predictor.18a9696e56738b8a namespace: kserve-ci-e2e-test resourceVersion: "19795" uid: 1cc236d3-eb45-4c55-8373-3b97ce5d0a21 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:51:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-1ac54-predictor namespace: kserve-ci-e2e-test resourceVersion: "19792" uid: 5d12a670-425c-49cf-bc38-9632204c57b8 kind: Event lastTimestamp: "2026-04-24T21:51:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:51:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:20Z" name: raw-sklearn-1ac54-predictor.18a96971d500a2d4 namespace: kserve-ci-e2e-test resourceVersion: "20060" uid: 7d6bd8ea-61e9-4ff9-b942-d715d7e47053 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:51:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-1ac54-predictor namespace: kserve-ci-e2e-test resourceVersion: "19792" uid: 5d12a670-425c-49cf-bc38-9632204c57b8 kind: Event lastTimestamp: "2026-04-24T21:51:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:51:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:20Z" name: raw-sklearn-1ac54-predictor.18a96971d56e96d2 namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 8f34bfee-3349-4d13-9b23-4a8e9fa8214d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:51:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-1ac54-predictor namespace: kserve-ci-e2e-test resourceVersion: "19956" uid: 5d12a670-425c-49cf-bc38-9632204c57b8 kind: Event lastTimestamp: "2026-04-24T21:52:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:52:05Z" name: raw-sklearn-1ac54-predictor.18a96978d3a50ae9 namespace: kserve-ci-e2e-test resourceVersion: "20385" uid: 3d57ec43-f0f2-444e-b890-43b36c1ec976 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:51:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-1ac54-predictor namespace: kserve-ci-e2e-test resourceVersion: "19956" uid: 5d12a670-425c-49cf-bc38-9632204c57b8 kind: Event lastTimestamp: "2026-04-24T21:52:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:52:05Z" name: raw-sklearn-1ac54-predictor.18a96978d41ea979 namespace: kserve-ci-e2e-test resourceVersion: "20386" uid: eb6f2496-a0a9-4c6e-95ed-b3e2702b7d52 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T21:50:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-1ac54 namespace: kserve-ci-e2e-test resourceVersion: "19779" uid: 609380e7-1443-40af-887b-56c350f3665f kind: Event lastTimestamp: "2026-04-24T21:52:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:52:03Z" name: raw-sklearn-1ac54.18a9696e564b85aa namespace: kserve-ci-e2e-test resourceVersion: "20367" uid: a5ee4caf-b0af-4527-8176-1e2ec7515543 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:50:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-1ac54 namespace: kserve-ci-e2e-test resourceVersion: "19779" uid: 609380e7-1443-40af-887b-56c350f3665f kind: Event lastTimestamp: "2026-04-24T21:50:51Z" message: 'Failed to update status for InferenceService "raw-sklearn-1ac54": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-1ac54": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:51Z" name: raw-sklearn-1ac54.18a9696e56c7fe35 namespace: kserve-ci-e2e-test resourceVersion: "19822" uid: 6b7e43f8-edbb-47ba-b304-1f1b80f8b5bd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-1ac54 namespace: kserve-ci-e2e-test resourceVersion: "19800" uid: 609380e7-1443-40af-887b-56c350f3665f kind: Event lastTimestamp: "2026-04-24T21:50:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-1ac54": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:51Z" name: raw-sklearn-1ac54.18a9696e70e428b7 namespace: kserve-ci-e2e-test resourceVersion: "19823" uid: 981c0a45-0d85-4778-9efd-c43297c53cc7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:52:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-1ac54 namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: 609380e7-1443-40af-887b-56c350f3665f kind: Event lastTimestamp: "2026-04-24T21:52:03Z" message: InferenceService [raw-sklearn-1ac54] is Ready metadata: creationTimestamp: "2026-04-24T21:52:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:52:03Z" name: raw-sklearn-1ac54.18a9697f4ea7812d namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: c9d010f1-96af-4634-ac97-e13128ffd649 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:52:10.934698Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7 namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: f59d6c94-3929-4f95-b202-40c968235482 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7 to ip-10-0-134-249.ec2.internal metadata: creationTimestamp: "2026-04-24T21:52:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:52:10Z" name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7.18a969810b3e5df2 namespace: kserve-ci-e2e-test resourceVersion: "20471" uid: c2cb43fa-a8cf-4afd-9d0e-920a7e3067af reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6688756845-wvlxd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:52:11Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7 namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: f59d6c94-3929-4f95-b202-40c968235482 kind: Event lastTimestamp: "2026-04-24T21:52:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-runtime-1f653-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:52:11Z" name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7.18a9698118c74eb1 namespace: kserve-ci-e2e-test resourceVersion: "20479" uid: f5542351-9870-492f-a050-46dd89022b27 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:52:11Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7 namespace: kserve-ci-e2e-test resourceVersion: "20475" uid: f59d6c94-3929-4f95-b202-40c968235482 kind: Event lastTimestamp: "2026-04-24T21:52:11Z" message: Add eth0 [10.132.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:52:11Z" name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7.18a9698147fef42b namespace: kserve-ci-e2e-test resourceVersion: "20487" uid: 1b173fa8-c132-462b-9378-74749e253505 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:52:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7 namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: f59d6c94-3929-4f95-b202-40c968235482 kind: Event lastTimestamp: "2026-04-24T21:52:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0dd2798f9cdbffc0563bd148612201df0e589ff8c26a6b19d321fd120fc5c097" already present on machine metadata: creationTimestamp: "2026-04-24T21:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:52:11Z" name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7.18a96981497cf071 namespace: kserve-ci-e2e-test resourceVersion: "20489" uid: a549c4b8-3a75-47e7-a038-8703010ebb73 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:52:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7 namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: f59d6c94-3929-4f95-b202-40c968235482 kind: Event lastTimestamp: "2026-04-24T21:52:12Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:52:12Z" name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7.18a969814c4065c7 namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: 589230d0-f7de-4a33-922e-2d7f77cd174f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:52:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7 namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: f59d6c94-3929-4f95-b202-40c968235482 kind: Event lastTimestamp: "2026-04-24T21:52:12Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:52:12Z" name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7.18a969814cacee2b namespace: kserve-ci-e2e-test resourceVersion: "20493" uid: 9f71e44f-aab5-4378-8f53-6136e9235a36 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:52:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7 namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: f59d6c94-3929-4f95-b202-40c968235482 kind: Event lastTimestamp: "2026-04-24T21:52:15Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:52:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:52:15Z" name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7.18a96982253eb211 namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: 702f417e-0190-4717-992b-067690808a85 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:52:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7 namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: f59d6c94-3929-4f95-b202-40c968235482 kind: Event lastTimestamp: "2026-04-24T21:52:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:52:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:52:15Z" name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7.18a9698228659c92 namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: bb5fd26a-8dfa-4028-bddc-f97d2af68e7a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:52:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7 namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: f59d6c94-3929-4f95-b202-40c968235482 kind: Event lastTimestamp: "2026-04-24T21:52:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:52:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:52:15Z" name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7.18a9698228d2e444 namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: e011bc2c-be50-41bb-b04e-e760756eb483 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:52:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7 namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: f59d6c94-3929-4f95-b202-40c968235482 kind: Event lastTimestamp: "2026-04-24T21:52:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:52:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:52:15Z" name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7.18a9698228df3f47 namespace: kserve-ci-e2e-test resourceVersion: "20533" uid: 4cf3db28-3b70-436e-800f-02911bea1b6c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:52:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7 namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: f59d6c94-3929-4f95-b202-40c968235482 kind: Event lastTimestamp: "2026-04-24T21:52:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:52:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:52:15Z" name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7.18a969822b84e95a namespace: kserve-ci-e2e-test resourceVersion: "20535" uid: 717ed69f-168c-43b6-af2c-49899136db0e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:52:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7 namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: f59d6c94-3929-4f95-b202-40c968235482 kind: Event lastTimestamp: "2026-04-24T21:52:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:52:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:52:15Z" name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7.18a969822bf2f134 namespace: kserve-ci-e2e-test resourceVersion: "20536" uid: 0f7c255c-9625-4333-9bb0-eb64ed66f174 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:52:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7 namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: f59d6c94-3929-4f95-b202-40c968235482 kind: Event lastTimestamp: "2026-04-24T21:53:32Z" message: 'Readiness probe failed: dial tcp 10.132.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:52:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:53:32Z" name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7.18a9698261360142 namespace: kserve-ci-e2e-test resourceVersion: "21144" uid: 1cb7442c-8071-4d1a-aed9-ccdaf8254b1b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:53:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7 namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: f59d6c94-3929-4f95-b202-40c968235482 kind: Event lastTimestamp: "2026-04-24T21:53:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:53:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:53:30Z" name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7.18a96993acd20c66 namespace: kserve-ci-e2e-test resourceVersion: "21101" uid: dfcd7949-c792-413c-ae34-60c7f500fce9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:53:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7 namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: f59d6c94-3929-4f95-b202-40c968235482 kind: Event lastTimestamp: "2026-04-24T21:53:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:53:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:53:30Z" name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7.18a96993acd277e0 namespace: kserve-ci-e2e-test resourceVersion: "21102" uid: 4785000d-0a87-493a-9ea1-d2efb8f15d31 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:53:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7 namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: f59d6c94-3929-4f95-b202-40c968235482 kind: Event lastTimestamp: "2026-04-24T21:53:32Z" message: 'Readiness probe failed: Get "https://10.132.0.46:8643/healthz": dial tcp 10.132.0.46:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:53:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:53:32Z" name: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7.18a969941356403e namespace: kserve-ci-e2e-test resourceVersion: "21143" uid: 59ae2f5a-5864-4f28-9fa8-597447b17a3d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-249.ec2.internal source: component: kubelet host: ip-10-0-134-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:52:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-1f653-predictor-7c88d7c964 namespace: kserve-ci-e2e-test resourceVersion: "20456" uid: 8b7979e1-f99f-4ef6-abe7-88ed9cab125f kind: Event lastTimestamp: "2026-04-24T21:52:10Z" message: 'Created pod: raw-sklearn-runtime-1f653-predictor-7c88d7c964-nc9t7' metadata: creationTimestamp: "2026-04-24T21:52:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:52:10Z" name: raw-sklearn-runtime-1f653-predictor-7c88d7c964.18a969810a9da5e3 namespace: kserve-ci-e2e-test resourceVersion: "20468" uid: 7d6d4169-3357-4602-ba06-0dedf977ef45 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:52:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-1f653-predictor namespace: kserve-ci-e2e-test resourceVersion: "20454" uid: 184b0fb0-afdd-43f4-bc2e-d4751077ddee kind: Event lastTimestamp: "2026-04-24T21:52:10Z" message: Scaled up replica set raw-sklearn-runtime-1f653-predictor-7c88d7c964 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:52:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:52:10Z" name: raw-sklearn-runtime-1f653-predictor.18a9698107674b97 namespace: kserve-ci-e2e-test resourceVersion: "20458" uid: 0d9f5122-9c32-4449-96ba-279bf30528f5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:52:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-1f653-predictor namespace: kserve-ci-e2e-test resourceVersion: "20455" uid: fc51edf3-fa65-4e5b-bddc-1f5f2a93e577 kind: Event lastTimestamp: "2026-04-24T21:52:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:52:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:52:40Z" name: raw-sklearn-runtime-1f653-predictor.18a9698486b3b679 namespace: kserve-ci-e2e-test resourceVersion: "20721" uid: 2b0173dc-ecad-40d8-8c69-0e7f296763a5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:52:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-1f653-predictor namespace: kserve-ci-e2e-test resourceVersion: "20455" uid: fc51edf3-fa65-4e5b-bddc-1f5f2a93e577 kind: Event lastTimestamp: "2026-04-24T21:52:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:52:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:52:40Z" name: raw-sklearn-runtime-1f653-predictor.18a969848735c008 namespace: kserve-ci-e2e-test resourceVersion: "20722" uid: 4893b91c-6d7e-49cc-95a0-9776bbeab726 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:52:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-1f653-predictor namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: fc51edf3-fa65-4e5b-bddc-1f5f2a93e577 kind: Event lastTimestamp: "2026-04-24T21:53:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:52:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:53:25Z" name: raw-sklearn-runtime-1f653-predictor.18a9698b854c0b93 namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: f321245d-55e2-4505-8653-57e2ff24a785 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:52:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-1f653-predictor namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: fc51edf3-fa65-4e5b-bddc-1f5f2a93e577 kind: Event lastTimestamp: "2026-04-24T21:53:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:52:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:53:25Z" name: raw-sklearn-runtime-1f653-predictor.18a9698b85c3dbe3 namespace: kserve-ci-e2e-test resourceVersion: "21046" uid: b81a6ec4-983f-4f95-a223-e7c6e9aab6a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T21:52:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-1f653 namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: 9d1c7609-6f88-46f4-9b71-96ad22fa28d5 kind: Event lastTimestamp: "2026-04-24T21:53:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:52:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:53:22Z" name: raw-sklearn-runtime-1f653.18a96981071465c7 namespace: kserve-ci-e2e-test resourceVersion: "21026" uid: 23bf6265-a321-4839-b99d-d63492336a24 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:52:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-1f653 namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: 9d1c7609-6f88-46f4-9b71-96ad22fa28d5 kind: Event lastTimestamp: "2026-04-24T21:52:10Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-1f653": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-1f653": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:52:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:52:10Z" name: raw-sklearn-runtime-1f653.18a969810776be3c namespace: kserve-ci-e2e-test resourceVersion: "20459" uid: c4092466-471e-4998-9e7a-e573c7d5769f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:53:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-1f653 namespace: kserve-ci-e2e-test resourceVersion: "21024" uid: 9d1c7609-6f88-46f4-9b71-96ad22fa28d5 kind: Event lastTimestamp: "2026-04-24T21:53:22Z" message: InferenceService [raw-sklearn-runtime-1f653] is Ready metadata: creationTimestamp: "2026-04-24T21:53:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:53:22Z" name: raw-sklearn-runtime-1f653.18a96991c4a8a7e3 namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: f62c230a-2325-41c9-a8ea-4f193497b2ff reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "21295"