--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:14:01.205858Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-8c321a-predictor-6cd87d9f66-z7rlh namespace: kserve-ci-e2e-test resourceVersion: "19642" uid: 8ab9a5f0-db9c-4c1b-ba7d-4ab756ac0d5b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-8c321a-predictor-6cd87d9f66-z7rlh to ip-10-0-142-46.ec2.internal metadata: creationTimestamp: "2026-04-16T15:14:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:14:01Z" name: isvc-init-fail-8c321a-predictor-6cd87d9f66-z7rlh.18a6df223df9335d namespace: kserve-ci-e2e-test resourceVersion: "19646" uid: f1c02dc0-61ea-41bc-a519-8edad45d4fd6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-794cffb559-pkg6f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:14:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-8c321a-predictor-6cd87d9f66-z7rlh namespace: kserve-ci-e2e-test resourceVersion: "19650" uid: 8ab9a5f0-db9c-4c1b-ba7d-4ab756ac0d5b kind: Event lastTimestamp: "2026-04-16T15:14:01Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:14:01Z" name: isvc-init-fail-8c321a-predictor-6cd87d9f66-z7rlh.18a6df2257c1f4da namespace: kserve-ci-e2e-test resourceVersion: "19660" uid: 223b305c-aeb2-4a43-bcb9-d6a600356d39 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:14:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-8c321a-predictor-6cd87d9f66-z7rlh namespace: kserve-ci-e2e-test resourceVersion: "19644" uid: 8ab9a5f0-db9c-4c1b-ba7d-4ab756ac0d5b kind: Event lastTimestamp: "2026-04-16T15:14:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:c49c84e69ddbb36db782821d4aa3d203f458a00c0c04dd506eb662aae339bb8a" already present on machine metadata: creationTimestamp: "2026-04-16T15:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:14:06Z" name: isvc-init-fail-8c321a-predictor-6cd87d9f66-z7rlh.18a6df2258f19a9d namespace: kserve-ci-e2e-test resourceVersion: "19710" uid: 1af1def6-d6d1-4b60-8e00-a92f81791263 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:14:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-8c321a-predictor-6cd87d9f66-z7rlh namespace: kserve-ci-e2e-test resourceVersion: "19644" uid: 8ab9a5f0-db9c-4c1b-ba7d-4ab756ac0d5b kind: Event lastTimestamp: "2026-04-16T15:14:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T15:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:14:06Z" name: isvc-init-fail-8c321a-predictor-6cd87d9f66-z7rlh.18a6df225bfa07af namespace: kserve-ci-e2e-test resourceVersion: "19716" uid: 111742d6-27a3-4f7e-b147-9e68ca50b323 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:14:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-8c321a-predictor-6cd87d9f66-z7rlh namespace: kserve-ci-e2e-test resourceVersion: "19644" uid: 8ab9a5f0-db9c-4c1b-ba7d-4ab756ac0d5b kind: Event lastTimestamp: "2026-04-16T15:14:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T15:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:14:06Z" name: isvc-init-fail-8c321a-predictor-6cd87d9f66-z7rlh.18a6df225c6f16a9 namespace: kserve-ci-e2e-test resourceVersion: "19717" uid: 1eaea62a-6d76-4d2e-8473-557c822abf6b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:14:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-8c321a-predictor-6cd87d9f66-z7rlh namespace: kserve-ci-e2e-test resourceVersion: "19644" uid: 8ab9a5f0-db9c-4c1b-ba7d-4ab756ac0d5b kind: Event lastTimestamp: "2026-04-16T15:14:09Z" message: Back-off restarting failed container storage-initializer in pod isvc-init-fail-8c321a-predictor-6cd87d9f66-z7rlh_kserve-ci-e2e-test(8ab9a5f0-db9c-4c1b-ba7d-4ab756ac0d5b) metadata: creationTimestamp: "2026-04-16T15:14:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:14:09Z" name: isvc-init-fail-8c321a-predictor-6cd87d9f66-z7rlh.18a6df241f657f31 namespace: kserve-ci-e2e-test resourceVersion: "19738" uid: a1c1be92-b7d9-4268-9ef0-c1663b810d5d reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:14:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-8c321a-predictor-6cd87d9f66 namespace: kserve-ci-e2e-test resourceVersion: "19631" uid: 344eb9b0-63c1-4b18-82ac-356260b093d0 kind: Event lastTimestamp: "2026-04-16T15:14:01Z" message: 'Created pod: isvc-init-fail-8c321a-predictor-6cd87d9f66-z7rlh' metadata: creationTimestamp: "2026-04-16T15:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:14:01Z" name: isvc-init-fail-8c321a-predictor-6cd87d9f66.18a6df223d8b5bbf namespace: kserve-ci-e2e-test resourceVersion: "19643" uid: a3448c13-cea7-4b81-92c8-8ad61e36e6f7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:14:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-8c321a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19627" uid: 2caec261-2d9b-428f-9937-9d48346f119c kind: Event lastTimestamp: "2026-04-16T15:14:01Z" message: Scaled up replica set isvc-init-fail-8c321a-predictor-6cd87d9f66 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:14:01Z" name: isvc-init-fail-8c321a-predictor.18a6df223ac85948 namespace: kserve-ci-e2e-test resourceVersion: "19633" uid: 5c1dbd58-ac9e-4eac-850d-869b80964de3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T15:14:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-8c321a namespace: kserve-ci-e2e-test resourceVersion: "19605" uid: 82c9afd1-4af5-4fe0-a3ad-2a8dbd16eec8 kind: Event lastTimestamp: "2026-04-16T15:14:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:14:10Z" name: isvc-init-fail-8c321a.18a6df223aa4edda namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: 6d47f937-97ce-4439-af8a-318d6742c598 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:14:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-8c321a namespace: kserve-ci-e2e-test resourceVersion: "19605" uid: 82c9afd1-4af5-4fe0-a3ad-2a8dbd16eec8 kind: Event lastTimestamp: "2026-04-16T15:14:01Z" message: 'Failed to update status for InferenceService "isvc-init-fail-8c321a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-8c321a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:14:01Z" name: isvc-init-fail-8c321a.18a6df223af6bb19 namespace: kserve-ci-e2e-test resourceVersion: "19635" uid: 3e9df25d-0f53-431d-86f1-b432b782f5cf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:14:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-8c321a namespace: kserve-ci-e2e-test resourceVersion: "19605" uid: 82c9afd1-4af5-4fe0-a3ad-2a8dbd16eec8 kind: Event lastTimestamp: "2026-04-16T15:14:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-8c321a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:14:01Z" name: isvc-init-fail-8c321a.18a6df223af7174d namespace: kserve-ci-e2e-test resourceVersion: "19636" uid: 9180a7ef-207d-4fbd-9f90-086c6699f67d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:08:29.359091Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-116da-predictor-679df64688-9nfhv namespace: kserve-ci-e2e-test resourceVersion: "16916" uid: 5d2d2964-af79-4836-958f-bed6d6dd4a47 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-116da-predictor-679df64688-9nfhv to ip-10-0-142-46.ec2.internal metadata: creationTimestamp: "2026-04-16T15:08:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:08:29Z" name: isvc-logger-raw-116da-predictor-679df64688-9nfhv.18a6ded4fa5d6910 namespace: kserve-ci-e2e-test resourceVersion: "16920" uid: 4bff38af-0d04-41a0-a365-a68097cb2eac reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-794cffb559-pkg6f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:29Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-116da-predictor-679df64688-9nfhv namespace: kserve-ci-e2e-test resourceVersion: "16925" uid: 5d2d2964-af79-4836-958f-bed6d6dd4a47 kind: Event lastTimestamp: "2026-04-16T15:08:29Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:08:29Z" name: isvc-logger-raw-116da-predictor-679df64688-9nfhv.18a6ded51333f6ce namespace: kserve-ci-e2e-test resourceVersion: "16935" uid: 748b6dad-cd05-458e-a2a7-80150b3e02fe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-116da-predictor-679df64688-9nfhv namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 5d2d2964-af79-4836-958f-bed6d6dd4a47 kind: Event lastTimestamp: "2026-04-16T15:08:29Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:c49c84e69ddbb36db782821d4aa3d203f458a00c0c04dd506eb662aae339bb8a" already present on machine metadata: creationTimestamp: "2026-04-16T15:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:29Z" name: isvc-logger-raw-116da-predictor-679df64688-9nfhv.18a6ded5148011ba namespace: kserve-ci-e2e-test resourceVersion: "16938" uid: 859fc111-529e-4f8e-86c9-36359ae8447f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-116da-predictor-679df64688-9nfhv namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 5d2d2964-af79-4836-958f-bed6d6dd4a47 kind: Event lastTimestamp: "2026-04-16T15:08:29Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T15:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:29Z" name: isvc-logger-raw-116da-predictor-679df64688-9nfhv.18a6ded517860db5 namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: 28942113-7a9d-4083-838f-64b1a041e011 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-116da-predictor-679df64688-9nfhv namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 5d2d2964-af79-4836-958f-bed6d6dd4a47 kind: Event lastTimestamp: "2026-04-16T15:08:29Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T15:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:29Z" name: isvc-logger-raw-116da-predictor-679df64688-9nfhv.18a6ded517edf6f8 namespace: kserve-ci-e2e-test resourceVersion: "16940" uid: 17bf960f-549a-46a8-b74d-b7a988b68bf3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-116da-predictor-679df64688-9nfhv namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 5d2d2964-af79-4836-958f-bed6d6dd4a47 kind: Event lastTimestamp: "2026-04-16T15:08:34Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:08:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:34Z" name: isvc-logger-raw-116da-predictor-679df64688-9nfhv.18a6ded61eea9f86 namespace: kserve-ci-e2e-test resourceVersion: "16976" uid: 2999cb9f-fa44-4e9e-880b-47af32944c4b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-116da-predictor-679df64688-9nfhv namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 5d2d2964-af79-4836-958f-bed6d6dd4a47 kind: Event lastTimestamp: "2026-04-16T15:08:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:08:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:34Z" name: isvc-logger-raw-116da-predictor-679df64688-9nfhv.18a6ded621c7f1e6 namespace: kserve-ci-e2e-test resourceVersion: "16983" uid: a29d35bb-64db-4407-8846-47c94c29d36e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-116da-predictor-679df64688-9nfhv namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 5d2d2964-af79-4836-958f-bed6d6dd4a47 kind: Event lastTimestamp: "2026-04-16T15:08:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:08:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:34Z" name: isvc-logger-raw-116da-predictor-679df64688-9nfhv.18a6ded62233d1f8 namespace: kserve-ci-e2e-test resourceVersion: "16984" uid: b2d00308-4f58-446d-a2e6-ea14f0554cfc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-116da-predictor-679df64688-9nfhv namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 5d2d2964-af79-4836-958f-bed6d6dd4a47 kind: Event lastTimestamp: "2026-04-16T15:08:34Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:93a3c88e5cf08b3e8030b29f6d098cd5406a6fd3ad769e6e697c2ae6303a7cab" already present on machine metadata: creationTimestamp: "2026-04-16T15:08:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:34Z" name: isvc-logger-raw-116da-predictor-679df64688-9nfhv.18a6ded62241c6a7 namespace: kserve-ci-e2e-test resourceVersion: "16985" uid: 1c903d74-bbba-4a6c-b555-8c3f02201bcb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-116da-predictor-679df64688-9nfhv namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 5d2d2964-af79-4836-958f-bed6d6dd4a47 kind: Event lastTimestamp: "2026-04-16T15:08:34Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-16T15:08:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:34Z" name: isvc-logger-raw-116da-predictor-679df64688-9nfhv.18a6ded626215337 namespace: kserve-ci-e2e-test resourceVersion: "16987" uid: 1948cfe3-f543-4241-a934-cdb84f3e82fa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-116da-predictor-679df64688-9nfhv namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 5d2d2964-af79-4836-958f-bed6d6dd4a47 kind: Event lastTimestamp: "2026-04-16T15:08:34Z" message: Started container agent metadata: creationTimestamp: "2026-04-16T15:08:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:34Z" name: isvc-logger-raw-116da-predictor-679df64688-9nfhv.18a6ded6268cb9d7 namespace: kserve-ci-e2e-test resourceVersion: "16988" uid: b51369dd-a850-4802-a294-23522698afe8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T15:08:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-116da-predictor-679df64688-9nfhv namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 5d2d2964-af79-4836-958f-bed6d6dd4a47 kind: Event lastTimestamp: "2026-04-16T15:10:17Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:10:17Z" name: isvc-logger-raw-116da-predictor-679df64688-9nfhv.18a6ded65ad5dd0d namespace: kserve-ci-e2e-test resourceVersion: "17838" uid: 07f8aaab-8728-4603-b6a1-5b3cd8bfd265 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-16T15:08:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-116da-predictor-679df64688-9nfhv namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 5d2d2964-af79-4836-958f-bed6d6dd4a47 kind: Event lastTimestamp: "2026-04-16T15:10:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T15:08:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:10:17Z" name: isvc-logger-raw-116da-predictor-679df64688-9nfhv.18a6ded696be32b9 namespace: kserve-ci-e2e-test resourceVersion: "17839" uid: 8e2f60e2-e8c3-4229-8ca7-9b7caf7fcc25 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:09:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-116da-predictor-679df64688-9nfhv namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 5d2d2964-af79-4836-958f-bed6d6dd4a47 kind: Event lastTimestamp: "2026-04-16T15:09:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:09:54Z" name: isvc-logger-raw-116da-predictor-679df64688-9nfhv.18a6dee8d1602fd3 namespace: kserve-ci-e2e-test resourceVersion: "17614" uid: e2f72223-36b6-48c9-a293-8c00cb18c1af reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:09:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-116da-predictor-679df64688-9nfhv namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 5d2d2964-af79-4836-958f-bed6d6dd4a47 kind: Event lastTimestamp: "2026-04-16T15:09:54Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-16T15:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:09:54Z" name: isvc-logger-raw-116da-predictor-679df64688-9nfhv.18a6dee8d1609a05 namespace: kserve-ci-e2e-test resourceVersion: "17615" uid: 098ccf6c-b9c4-4f68-b0b0-e84addbf4020 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-116da-predictor-679df64688 namespace: kserve-ci-e2e-test resourceVersion: "16903" uid: c4874ef5-2137-4f5c-a2d7-60c740587978 kind: Event lastTimestamp: "2026-04-16T15:08:29Z" message: 'Created pod: isvc-logger-raw-116da-predictor-679df64688-9nfhv' metadata: creationTimestamp: "2026-04-16T15:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:29Z" name: isvc-logger-raw-116da-predictor-679df64688.18a6ded4f9f29790 namespace: kserve-ci-e2e-test resourceVersion: "16917" uid: f6d7d66b-7661-4156-a4ea-d14050ab808f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-116da-predictor namespace: kserve-ci-e2e-test resourceVersion: "16901" uid: ce291cdf-eb4c-4523-a6bf-540c719161fe kind: Event lastTimestamp: "2026-04-16T15:08:29Z" message: Scaled up replica set isvc-logger-raw-116da-predictor-679df64688 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:29Z" name: isvc-logger-raw-116da-predictor.18a6ded4f74e50f4 namespace: kserve-ci-e2e-test resourceVersion: "16906" uid: cd2e1aae-0a43-416d-a3c8-5d254b8709c8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:08:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-116da-predictor namespace: kserve-ci-e2e-test resourceVersion: "16902" uid: e04b138b-e022-4068-9495-fc4fe0ed6110 kind: Event lastTimestamp: "2026-04-16T15:09:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:09:14Z" name: isvc-logger-raw-116da-predictor.18a6ded875c4f519 namespace: kserve-ci-e2e-test resourceVersion: "17286" uid: ee81bcde-12f2-470f-b70d-6209f2297177 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:08:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-116da-predictor namespace: kserve-ci-e2e-test resourceVersion: "16902" uid: e04b138b-e022-4068-9495-fc4fe0ed6110 kind: Event lastTimestamp: "2026-04-16T15:09:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:09:14Z" name: isvc-logger-raw-116da-predictor.18a6ded876237a63 namespace: kserve-ci-e2e-test resourceVersion: "17287" uid: 5342030e-1ef3-4524-a9e2-0061c4e57a47 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:08:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-116da-predictor namespace: kserve-ci-e2e-test resourceVersion: "17066" uid: e04b138b-e022-4068-9495-fc4fe0ed6110 kind: Event lastTimestamp: "2026-04-16T15:09:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:09:44Z" name: isvc-logger-raw-116da-predictor.18a6dedbf4ce7d31 namespace: kserve-ci-e2e-test resourceVersion: "17501" uid: 1ce8a2d8-0888-4915-a46e-d30638378035 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:08:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-116da-predictor namespace: kserve-ci-e2e-test resourceVersion: "17066" uid: e04b138b-e022-4068-9495-fc4fe0ed6110 kind: Event lastTimestamp: "2026-04-16T15:09:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:09:44Z" name: isvc-logger-raw-116da-predictor.18a6dedbf531c7ae namespace: kserve-ci-e2e-test resourceVersion: "17502" uid: 1d276622-7f56-45c4-b7a5-efa4159504f2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T15:08:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-116da namespace: kserve-ci-e2e-test resourceVersion: "16893" uid: c6e164a9-1f08-4005-b16f-11f03aa659b5 kind: Event lastTimestamp: "2026-04-16T15:09:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:09:47Z" name: isvc-logger-raw-116da.18a6ded4f72945e9 namespace: kserve-ci-e2e-test resourceVersion: "17533" uid: 76f4c58a-991c-4689-8f44-2ab2851fd544 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:08:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-116da namespace: kserve-ci-e2e-test resourceVersion: "16893" uid: c6e164a9-1f08-4005-b16f-11f03aa659b5 kind: Event lastTimestamp: "2026-04-16T15:08:29Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-116da": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-116da": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:08:29Z" name: isvc-logger-raw-116da.18a6ded4f7be8546 namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: 9dc5bb7b-7df4-4665-b9b9-762fff211860 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-116da namespace: kserve-ci-e2e-test resourceVersion: "16898" uid: c6e164a9-1f08-4005-b16f-11f03aa659b5 kind: Event lastTimestamp: "2026-04-16T15:08:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-116da": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:08:29Z" name: isvc-logger-raw-116da.18a6ded4fbdeef1c namespace: kserve-ci-e2e-test resourceVersion: "16927" uid: b7bcfafd-204b-48fe-93ae-713c40cb9618 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:09:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-116da namespace: kserve-ci-e2e-test resourceVersion: "17529" uid: c6e164a9-1f08-4005-b16f-11f03aa659b5 kind: Event lastTimestamp: "2026-04-16T15:09:47Z" message: InferenceService [isvc-logger-raw-116da] is Ready metadata: creationTimestamp: "2026-04-16T15:09:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:09:47Z" name: isvc-logger-raw-116da.18a6dee7227fa9e7 namespace: kserve-ci-e2e-test resourceVersion: "17532" uid: d64523b3-4442-435b-b178-e15b66f11f80 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:12:24.883358Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp namespace: kserve-ci-e2e-test resourceVersion: "18794" uid: ecd02460-5936-48c3-b169-fd8a14fe81d7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp to ip-10-0-142-46.ec2.internal metadata: creationTimestamp: "2026-04-16T15:12:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:12:24Z" name: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp.18a6df0bd0b483ee namespace: kserve-ci-e2e-test resourceVersion: "18798" uid: f48928a2-6213-4e3b-9554-6e9ed4e79117 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-794cffb559-pkg6f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:12:25Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp namespace: kserve-ci-e2e-test resourceVersion: "18802" uid: ecd02460-5936-48c3-b169-fd8a14fe81d7 kind: Event lastTimestamp: "2026-04-16T15:12:25Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:12:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:12:25Z" name: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp.18a6df0be9cf1e37 namespace: kserve-ci-e2e-test resourceVersion: "18809" uid: 17e1eb86-50c7-423b-80b8-e4aa94634a30 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:12:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp namespace: kserve-ci-e2e-test resourceVersion: "18796" uid: ecd02460-5936-48c3-b169-fd8a14fe81d7 kind: Event lastTimestamp: "2026-04-16T15:12:25Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:c49c84e69ddbb36db782821d4aa3d203f458a00c0c04dd506eb662aae339bb8a" already present on machine metadata: creationTimestamp: "2026-04-16T15:12:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:12:25Z" name: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp.18a6df0beb082771 namespace: kserve-ci-e2e-test resourceVersion: "18811" uid: 78d537d6-c414-440d-ae30-20c6f8da7518 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:12:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp namespace: kserve-ci-e2e-test resourceVersion: "18796" uid: ecd02460-5936-48c3-b169-fd8a14fe81d7 kind: Event lastTimestamp: "2026-04-16T15:12:25Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T15:12:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:12:25Z" name: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp.18a6df0bede7f10e namespace: kserve-ci-e2e-test resourceVersion: "18815" uid: 1c6e1476-9ecb-4a74-ba0e-781945ee5d37 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:12:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp namespace: kserve-ci-e2e-test resourceVersion: "18796" uid: ecd02460-5936-48c3-b169-fd8a14fe81d7 kind: Event lastTimestamp: "2026-04-16T15:12:25Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T15:12:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:12:25Z" name: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp.18a6df0bee5dd4f9 namespace: kserve-ci-e2e-test resourceVersion: "18816" uid: 221f495a-ed50-4090-931b-1c7738106228 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:12:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp namespace: kserve-ci-e2e-test resourceVersion: "18796" uid: ecd02460-5936-48c3-b169-fd8a14fe81d7 kind: Event lastTimestamp: "2026-04-16T15:12:29Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:12:29Z" name: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp.18a6df0d003339db namespace: kserve-ci-e2e-test resourceVersion: "18849" uid: 76071d30-129f-4172-9d2e-dca3d81f2090 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:12:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp namespace: kserve-ci-e2e-test resourceVersion: "18796" uid: ecd02460-5936-48c3-b169-fd8a14fe81d7 kind: Event lastTimestamp: "2026-04-16T15:12:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:12:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:12:30Z" name: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp.18a6df0d0309e155 namespace: kserve-ci-e2e-test resourceVersion: "18853" uid: 72d3201c-874d-428b-9acd-b801b21b38a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:12:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp namespace: kserve-ci-e2e-test resourceVersion: "18796" uid: ecd02460-5936-48c3-b169-fd8a14fe81d7 kind: Event lastTimestamp: "2026-04-16T15:12:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:12:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:12:30Z" name: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp.18a6df0d037efdb4 namespace: kserve-ci-e2e-test resourceVersion: "18854" uid: 8a4a3b1b-e4a6-4b51-8c92-f900e09f306a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T15:12:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp namespace: kserve-ci-e2e-test resourceVersion: "18796" uid: ecd02460-5936-48c3-b169-fd8a14fe81d7 kind: Event lastTimestamp: "2026-04-16T15:13:31Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:12:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:13:31Z" name: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp.18a6df0d3c1dbccd namespace: kserve-ci-e2e-test resourceVersion: "19295" uid: b06cdeff-c41d-48bd-ab61-346172c6060f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:14:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp namespace: kserve-ci-e2e-test resourceVersion: "18796" uid: ecd02460-5936-48c3-b169-fd8a14fe81d7 kind: Event lastTimestamp: "2026-04-16T15:14:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:14:01Z" name: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp.18a6df223a2e123e namespace: kserve-ci-e2e-test resourceVersion: "19628" uid: ecf6a790-66c4-4202-b6c1-6192f28109f4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:12:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-f9fec2-predictor-784f9bbf45 namespace: kserve-ci-e2e-test resourceVersion: "18782" uid: 235954c1-110f-470c-abb3-5e048c62d918 kind: Event lastTimestamp: "2026-04-16T15:12:24Z" message: 'Created pod: isvc-primary-f9fec2-predictor-784f9bbf45-cgzlp' metadata: creationTimestamp: "2026-04-16T15:12:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:12:24Z" name: isvc-primary-f9fec2-predictor-784f9bbf45.18a6df0bd04e66c3 namespace: kserve-ci-e2e-test resourceVersion: "18795" uid: 83a5c792-7c74-44b8-9abe-17e0a97fc987 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:12:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-f9fec2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18778" uid: 1fd044fa-6cef-4259-9a20-68ffa8ed8487 kind: Event lastTimestamp: "2026-04-16T15:12:24Z" message: Scaled up replica set isvc-primary-f9fec2-predictor-784f9bbf45 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:12:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:12:24Z" name: isvc-primary-f9fec2-predictor.18a6df0bcbf91a5a namespace: kserve-ci-e2e-test resourceVersion: "18786" uid: 348002e6-a9ec-4a95-8cef-413d543ea327 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:12:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-f9fec2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18781" uid: f78b79e9-ad22-4808-8f43-e610cb1141cd kind: Event lastTimestamp: "2026-04-16T15:12:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:12:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:12:54Z" name: isvc-primary-f9fec2-predictor.18a6df0f4ab180b5 namespace: kserve-ci-e2e-test resourceVersion: "19036" uid: fb232147-f75f-4c72-a564-3b90eee7fa48 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:12:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-f9fec2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18781" uid: f78b79e9-ad22-4808-8f43-e610cb1141cd kind: Event lastTimestamp: "2026-04-16T15:12:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:12:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:12:54Z" name: isvc-primary-f9fec2-predictor.18a6df0f4aee8682 namespace: kserve-ci-e2e-test resourceVersion: "19037" uid: cd24db39-e807-443c-a9d4-7405ba8dc061 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T15:13:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-f9fec2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18933" uid: f78b79e9-ad22-4808-8f43-e610cb1141cd kind: Event lastTimestamp: "2026-04-16T15:13:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:13:54Z" name: isvc-primary-f9fec2-predictor.18a6df1648c288fe namespace: kserve-ci-e2e-test resourceVersion: "19529" uid: 8103c868-8f0b-424b-bd4d-1380d599393e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T15:13:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-f9fec2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18933" uid: f78b79e9-ad22-4808-8f43-e610cb1141cd kind: Event lastTimestamp: "2026-04-16T15:13:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:13:54Z" name: isvc-primary-f9fec2-predictor.18a6df1649260983 namespace: kserve-ci-e2e-test resourceVersion: "19530" uid: 1df36e13-d4ea-41d7-9fe1-7d60cb78bc67 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T15:12:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-f9fec2 namespace: kserve-ci-e2e-test resourceVersion: "18755" uid: 5207c1f8-6d7b-4ab7-b147-5bab506bccfb kind: Event lastTimestamp: "2026-04-16T15:13:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:12:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:13:39Z" name: isvc-primary-f9fec2.18a6df0bcbd39378 namespace: kserve-ci-e2e-test resourceVersion: "19354" uid: db26c6b9-c8eb-4d6d-a69f-cbe8c8d87266 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:12:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-f9fec2 namespace: kserve-ci-e2e-test resourceVersion: "18755" uid: 5207c1f8-6d7b-4ab7-b147-5bab506bccfb kind: Event lastTimestamp: "2026-04-16T15:12:24Z" message: 'Failed to update status for InferenceService "isvc-primary-f9fec2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-f9fec2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:12:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:12:24Z" name: isvc-primary-f9fec2.18a6df0bcc403a7b namespace: kserve-ci-e2e-test resourceVersion: "18788" uid: 55aa7944-99f2-496f-9507-369977ceec00 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:12:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-f9fec2 namespace: kserve-ci-e2e-test resourceVersion: "18755" uid: 5207c1f8-6d7b-4ab7-b147-5bab506bccfb kind: Event lastTimestamp: "2026-04-16T15:12:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-f9fec2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:12:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:12:24Z" name: isvc-primary-f9fec2.18a6df0bcc408145 namespace: kserve-ci-e2e-test resourceVersion: "18789" uid: 6aa777b3-397c-498c-a3e4-f4f821704d26 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:13:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-f9fec2 namespace: kserve-ci-e2e-test resourceVersion: "19352" uid: 5207c1f8-6d7b-4ab7-b147-5bab506bccfb kind: Event lastTimestamp: "2026-04-16T15:13:39Z" message: InferenceService [isvc-primary-f9fec2] is Ready metadata: creationTimestamp: "2026-04-16T15:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:13:39Z" name: isvc-primary-f9fec2.18a6df1d2963be8a namespace: kserve-ci-e2e-test resourceVersion: "19353" uid: 2d0eb78e-8af7-4883-bbda-12f26e8a54a7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:03:13.552266Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2 namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: 4e64adc3-bb32-4063-9c38-b687be03f0da kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2 to ip-10-0-142-46.ec2.internal metadata: creationTimestamp: "2026-04-16T15:03:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:03:13Z" name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2.18a6de8b72cfb32c namespace: kserve-ci-e2e-test resourceVersion: "14018" uid: aa600779-2293-4d59-92c9-207e61869793 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-794cffb559-pkg6f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:03:13Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2 namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: 4e64adc3-bb32-4063-9c38-b687be03f0da kind: Event lastTimestamp: "2026-04-16T15:03:13Z" message: Add eth0 [10.133.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:03:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:03:13Z" name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2.18a6de8b8c69e0dd namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: 21d78542-ce77-4c58-af08-ed3be21c3b22 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:03:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2 namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 4e64adc3-bb32-4063-9c38-b687be03f0da kind: Event lastTimestamp: "2026-04-16T15:03:14Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:c49c84e69ddbb36db782821d4aa3d203f458a00c0c04dd506eb662aae339bb8a" metadata: creationTimestamp: "2026-04-16T15:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:03:14Z" name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2.18a6de8b8f499bec namespace: kserve-ci-e2e-test resourceVersion: "14040" uid: 548ee3ce-c244-4435-8cc5-f364d184495d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:03:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2 namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 4e64adc3-bb32-4063-9c38-b687be03f0da kind: Event lastTimestamp: "2026-04-16T15:03:18Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:c49c84e69ddbb36db782821d4aa3d203f458a00c0c04dd506eb662aae339bb8a" in 4.475s (4.475s including waiting). Image size: 300960724 bytes.' metadata: creationTimestamp: "2026-04-16T15:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:03:18Z" name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2.18a6de8c9a08f563 namespace: kserve-ci-e2e-test resourceVersion: "14076" uid: 6cc4bb21-3b8c-4d93-88f9-e5d30319e1ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:03:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2 namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 4e64adc3-bb32-4063-9c38-b687be03f0da kind: Event lastTimestamp: "2026-04-16T15:03:18Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T15:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:03:18Z" name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2.18a6de8c9d3cdd99 namespace: kserve-ci-e2e-test resourceVersion: "14078" uid: d887900b-66a8-40cb-b259-58ad6f77882a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:03:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2 namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 4e64adc3-bb32-4063-9c38-b687be03f0da kind: Event lastTimestamp: "2026-04-16T15:03:18Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T15:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:03:18Z" name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2.18a6de8c9da589d6 namespace: kserve-ci-e2e-test resourceVersion: "14079" uid: 32641a40-17e3-4755-ac71-78c65dda05b2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:03:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2 namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 4e64adc3-bb32-4063-9c38-b687be03f0da kind: Event lastTimestamp: "2026-04-16T15:03:23Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1407" metadata: creationTimestamp: "2026-04-16T15:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:03:23Z" name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2.18a6de8db76e0789 namespace: kserve-ci-e2e-test resourceVersion: "14116" uid: 2acd88e7-3d70-4852-b348-767099bfff44 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:03:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2 namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 4e64adc3-bb32-4063-9c38-b687be03f0da kind: Event lastTimestamp: "2026-04-16T15:03:37Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1407" in 14.082s (14.082s including waiting). Image size: 1560613827 bytes.' metadata: creationTimestamp: "2026-04-16T15:03:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:03:37Z" name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2.18a6de90fecf4e03 namespace: kserve-ci-e2e-test resourceVersion: "14220" uid: 15f0f8c3-1bf8-442d-8cd5-421305933b1f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:03:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2 namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 4e64adc3-bb32-4063-9c38-b687be03f0da kind: Event lastTimestamp: "2026-04-16T15:03:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:03:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:03:37Z" name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2.18a6de91020a7eee namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: 83977cd4-b33d-48c3-8d36-0ce8266fca37 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:03:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2 namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 4e64adc3-bb32-4063-9c38-b687be03f0da kind: Event lastTimestamp: "2026-04-16T15:03:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:03:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:03:37Z" name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2.18a6de910270c166 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 1ba780dd-8e96-4917-9abd-d925d48e6d84 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:03:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2 namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 4e64adc3-bb32-4063-9c38-b687be03f0da kind: Event lastTimestamp: "2026-04-16T15:03:37Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:93a3c88e5cf08b3e8030b29f6d098cd5406a6fd3ad769e6e697c2ae6303a7cab" metadata: creationTimestamp: "2026-04-16T15:03:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:03:37Z" name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2.18a6de91027dcbbc namespace: kserve-ci-e2e-test resourceVersion: "14223" uid: 0a428071-3d46-4f3b-94cb-4053214bd1c4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:03:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2 namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 4e64adc3-bb32-4063-9c38-b687be03f0da kind: Event lastTimestamp: "2026-04-16T15:03:41Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:93a3c88e5cf08b3e8030b29f6d098cd5406a6fd3ad769e6e697c2ae6303a7cab" in 3.82s (3.82s including waiting). Image size: 237895798 bytes.' metadata: creationTimestamp: "2026-04-16T15:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:03:41Z" name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2.18a6de91e632b6ba namespace: kserve-ci-e2e-test resourceVersion: "14250" uid: 3eff27b8-a175-4afa-97a2-e016b77e1ec2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:03:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2 namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 4e64adc3-bb32-4063-9c38-b687be03f0da kind: Event lastTimestamp: "2026-04-16T15:03:41Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-16T15:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:03:41Z" name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2.18a6de91e8e3531a namespace: kserve-ci-e2e-test resourceVersion: "14251" uid: ee7e3963-676d-4a25-aa8b-d9efb38b7e57 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:03:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2 namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 4e64adc3-bb32-4063-9c38-b687be03f0da kind: Event lastTimestamp: "2026-04-16T15:03:41Z" message: Started container agent metadata: creationTimestamp: "2026-04-16T15:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:03:41Z" name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2.18a6de91e9483c0b namespace: kserve-ci-e2e-test resourceVersion: "14252" uid: e9e5098b-c9ec-4bab-872b-8e819167bf09 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-16T15:03:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2 namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 4e64adc3-bb32-4063-9c38-b687be03f0da kind: Event lastTimestamp: "2026-04-16T15:05:23Z" message: 'Readiness probe failed: dial tcp 10.133.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:23Z" name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2.18a6de91ec012fb0 namespace: kserve-ci-e2e-test resourceVersion: "15127" uid: 33b3d450-e44e-45f9-bd9b-8fea0080edca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T15:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2 namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 4e64adc3-bb32-4063-9c38-b687be03f0da kind: Event lastTimestamp: "2026-04-16T15:05:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T15:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:23Z" name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2.18a6de9227c35a50 namespace: kserve-ci-e2e-test resourceVersion: "15128" uid: 6282c7b1-19a6-40b5-a37d-45ae45867aa0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:04:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2 namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 4e64adc3-bb32-4063-9c38-b687be03f0da kind: Event lastTimestamp: "2026-04-16T15:04:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:04:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:04:58Z" name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2.18a6dea3eee10736 namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 7833dd30-1d27-4571-8871-9405cb6aaa5f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:04:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2 namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 4e64adc3-bb32-4063-9c38-b687be03f0da kind: Event lastTimestamp: "2026-04-16T15:04:58Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-16T15:04:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:04:58Z" name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2.18a6dea3eee248e8 namespace: kserve-ci-e2e-test resourceVersion: "14847" uid: 335f814b-4ccd-4372-b9d7-3463bceb6532 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:03:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b namespace: kserve-ci-e2e-test resourceVersion: "14003" uid: 92ed7396-154d-4773-9e56-220c3135234c kind: Event lastTimestamp: "2026-04-16T15:03:13Z" message: 'Created pod: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b-xn5s2' metadata: creationTimestamp: "2026-04-16T15:03:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:03:13Z" name: isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b.18a6de8b724bd21f namespace: kserve-ci-e2e-test resourceVersion: "14014" uid: 46cac04e-7415-4f0b-9cf7-6253f15ab905 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:03:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-f99a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14002" uid: 23e24a0d-9fcb-49d2-b5b8-e84519cbdfe1 kind: Event lastTimestamp: "2026-04-16T15:03:13Z" message: Scaled up replica set isvc-raw-sklearn-batcher-f99a6-predictor-698b5c9d6b from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:03:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:03:13Z" name: isvc-raw-sklearn-batcher-f99a6-predictor.18a6de8b6bb0b9cc namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 22aa7551-633c-46f2-aeab-783ed7972f3c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:03:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-f99a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14010" uid: e07c9f31-ec8c-489f-841b-55cffcac5fcf kind: Event lastTimestamp: "2026-04-16T15:03:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:03:43Z" name: isvc-raw-sklearn-batcher-f99a6-predictor.18a6de8ef0a6d249 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: fd8c25e4-06df-4120-beec-8e9d8f6e8285 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:03:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-f99a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14010" uid: e07c9f31-ec8c-489f-841b-55cffcac5fcf kind: Event lastTimestamp: "2026-04-16T15:03:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:03:43Z" name: isvc-raw-sklearn-batcher-f99a6-predictor.18a6de8ef10cb52c namespace: kserve-ci-e2e-test resourceVersion: "14277" uid: ef8c5306-ce92-4c9c-afcc-63ba06accd98 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:03:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-f99a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14157" uid: e07c9f31-ec8c-489f-841b-55cffcac5fcf kind: Event lastTimestamp: "2026-04-16T15:04:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:03:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:04:58Z" name: isvc-raw-sklearn-batcher-f99a6-predictor.18a6de95eef5c0f2 namespace: kserve-ci-e2e-test resourceVersion: "14823" uid: e4748ed9-7ab8-44c2-be69-6059f14ee239 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:03:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-f99a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14157" uid: e07c9f31-ec8c-489f-841b-55cffcac5fcf kind: Event lastTimestamp: "2026-04-16T15:04:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:03:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:04:58Z" name: isvc-raw-sklearn-batcher-f99a6-predictor.18a6de95ef5502cb namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 24ea57ab-ff2a-432f-83a5-9b98f808dde9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 15 eventTime: null firstTimestamp: "2026-04-16T15:03:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-f99a6 namespace: kserve-ci-e2e-test resourceVersion: "13991" uid: d4798a4e-8c97-4e8f-ba03-7d658a87bba6 kind: Event lastTimestamp: "2026-04-16T15:04:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:03:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:04:43Z" name: isvc-raw-sklearn-batcher-f99a6.18a6de8b71b79a43 namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: 87c4f846-7086-4fc7-b726-743eb4643294 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:03:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-f99a6 namespace: kserve-ci-e2e-test resourceVersion: "13991" uid: d4798a4e-8c97-4e8f-ba03-7d658a87bba6 kind: Event lastTimestamp: "2026-04-16T15:03:13Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-f99a6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-f99a6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:03:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:03:13Z" name: isvc-raw-sklearn-batcher-f99a6.18a6de8b72250495 namespace: kserve-ci-e2e-test resourceVersion: "14026" uid: a08d0e4d-575b-4bb8-92d0-4f9c8fdb9d5b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:03:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-f99a6 namespace: kserve-ci-e2e-test resourceVersion: "13991" uid: d4798a4e-8c97-4e8f-ba03-7d658a87bba6 kind: Event lastTimestamp: "2026-04-16T15:03:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-f99a6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:03:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:03:13Z" name: isvc-raw-sklearn-batcher-f99a6.18a6de8b722555f8 namespace: kserve-ci-e2e-test resourceVersion: "14027" uid: c08c5a02-39b0-48e7-911a-b9a6c5b64d30 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:04:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-f99a6 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: d4798a4e-8c97-4e8f-ba03-7d658a87bba6 kind: Event lastTimestamp: "2026-04-16T15:04:43Z" message: InferenceService [isvc-raw-sklearn-batcher-f99a6] is Ready metadata: creationTimestamp: "2026-04-16T15:04:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:04:43Z" name: isvc-raw-sklearn-batcher-f99a6.18a6dea060d6d2e8 namespace: kserve-ci-e2e-test resourceVersion: "14705" uid: e0d9b028-8044-4b6f-845d-cee2fa91df50 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:13:44.996746Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-f9fec2-predictor-56fc78594d-hj4lv namespace: kserve-ci-e2e-test resourceVersion: "19423" uid: 71dbcd77-e56b-47d2-9890-a4892849c4b5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-f9fec2-predictor-56fc78594d-hj4lv to ip-10-0-142-46.ec2.internal metadata: creationTimestamp: "2026-04-16T15:13:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:13:44Z" name: isvc-secondary-f9fec2-predictor-56fc78594d-hj4lv.18a6df1e77d5d53b namespace: kserve-ci-e2e-test resourceVersion: "19428" uid: 93babe1a-c68a-4da9-bced-f8aaf5683346 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-794cffb559-pkg6f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:13:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-f9fec2-predictor-56fc78594d-hj4lv namespace: kserve-ci-e2e-test resourceVersion: "19431" uid: 71dbcd77-e56b-47d2-9890-a4892849c4b5 kind: Event lastTimestamp: "2026-04-16T15:13:45Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:13:45Z" name: isvc-secondary-f9fec2-predictor-56fc78594d-hj4lv.18a6df1e9127d897 namespace: kserve-ci-e2e-test resourceVersion: "19442" uid: 0985efa0-63e6-4417-bfb1-e4a0ad88df5c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:13:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-f9fec2-predictor-56fc78594d-hj4lv namespace: kserve-ci-e2e-test resourceVersion: "19425" uid: 71dbcd77-e56b-47d2-9890-a4892849c4b5 kind: Event lastTimestamp: "2026-04-16T15:13:49Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:c49c84e69ddbb36db782821d4aa3d203f458a00c0c04dd506eb662aae339bb8a" already present on machine metadata: creationTimestamp: "2026-04-16T15:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:13:49Z" name: isvc-secondary-f9fec2-predictor-56fc78594d-hj4lv.18a6df1e92514a31 namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: a1eb4d80-90cb-4bfd-b56f-87f3c6129236 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:13:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-f9fec2-predictor-56fc78594d-hj4lv namespace: kserve-ci-e2e-test resourceVersion: "19425" uid: 71dbcd77-e56b-47d2-9890-a4892849c4b5 kind: Event lastTimestamp: "2026-04-16T15:13:49Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T15:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:13:49Z" name: isvc-secondary-f9fec2-predictor-56fc78594d-hj4lv.18a6df1e957b16c7 namespace: kserve-ci-e2e-test resourceVersion: "19483" uid: 9c136bfa-d9c0-41cc-838c-8816d5d3a576 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:13:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-f9fec2-predictor-56fc78594d-hj4lv namespace: kserve-ci-e2e-test resourceVersion: "19425" uid: 71dbcd77-e56b-47d2-9890-a4892849c4b5 kind: Event lastTimestamp: "2026-04-16T15:13:49Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T15:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:13:49Z" name: isvc-secondary-f9fec2-predictor-56fc78594d-hj4lv.18a6df1e95eb5b70 namespace: kserve-ci-e2e-test resourceVersion: "19484" uid: bfd858fe-de32-43b0-b832-db2895317281 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:13:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-f9fec2-predictor-56fc78594d-hj4lv namespace: kserve-ci-e2e-test resourceVersion: "19425" uid: 71dbcd77-e56b-47d2-9890-a4892849c4b5 kind: Event lastTimestamp: "2026-04-16T15:13:53Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-f9fec2-predictor-56fc78594d-hj4lv_kserve-ci-e2e-test(71dbcd77-e56b-47d2-9890-a4892849c4b5) metadata: creationTimestamp: "2026-04-16T15:13:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:13:53Z" name: isvc-secondary-f9fec2-predictor-56fc78594d-hj4lv.18a6df20622e4e76 namespace: kserve-ci-e2e-test resourceVersion: "19515" uid: 9cd92433-c952-4fd7-bfa3-a25903da12ad reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:13:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-f9fec2-predictor-56fc78594d namespace: kserve-ci-e2e-test resourceVersion: "19412" uid: e345e34c-e582-4489-9bbd-8724cae1106b kind: Event lastTimestamp: "2026-04-16T15:13:44Z" message: 'Created pod: isvc-secondary-f9fec2-predictor-56fc78594d-hj4lv' metadata: creationTimestamp: "2026-04-16T15:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:13:44Z" name: isvc-secondary-f9fec2-predictor-56fc78594d.18a6df1e777107c7 namespace: kserve-ci-e2e-test resourceVersion: "19424" uid: 2e214f02-4325-499b-88f7-5a7a0fa6a83c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:13:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-f9fec2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19410" uid: a80ba87d-b676-427c-bb8e-d4abf0104764 kind: Event lastTimestamp: "2026-04-16T15:13:44Z" message: Scaled up replica set isvc-secondary-f9fec2-predictor-56fc78594d from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:13:44Z" name: isvc-secondary-f9fec2-predictor.18a6df1e74b6b547 namespace: kserve-ci-e2e-test resourceVersion: "19415" uid: 08966791-7e3c-4171-9138-4e493c6b997c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:13:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-f9fec2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19413" uid: 99851458-929f-4851-9bcd-5a90b02a65ef kind: Event lastTimestamp: "2026-04-16T15:13:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:13:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:13:59Z" name: isvc-secondary-f9fec2-predictor.18a6df21f347b0e5 namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: 440eb56f-7602-4292-8f23-307546388925 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:13:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-f9fec2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19413" uid: 99851458-929f-4851-9bcd-5a90b02a65ef kind: Event lastTimestamp: "2026-04-16T15:13:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:13:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:13:59Z" name: isvc-secondary-f9fec2-predictor.18a6df21f3a39ffb namespace: kserve-ci-e2e-test resourceVersion: "19572" uid: 8f0fa236-5793-4dd2-8113-55a4fb782133 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-04-16T15:13:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-f9fec2 namespace: kserve-ci-e2e-test resourceVersion: "19402" uid: cc8bb124-2250-43e9-8b31-91242f5f195a kind: Event lastTimestamp: "2026-04-16T15:13:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:13:55Z" name: isvc-secondary-f9fec2.18a6df1e74be55cf namespace: kserve-ci-e2e-test resourceVersion: "19536" uid: 94382959-ffcf-4a9e-b044-4db31ece8813 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:13:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-f9fec2 namespace: kserve-ci-e2e-test resourceVersion: "19402" uid: cc8bb124-2250-43e9-8b31-91242f5f195a kind: Event lastTimestamp: "2026-04-16T15:13:45Z" message: 'Failed to update status for InferenceService "isvc-secondary-f9fec2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-f9fec2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:13:45Z" name: isvc-secondary-f9fec2.18a6df1e7524daef namespace: kserve-ci-e2e-test resourceVersion: "19433" uid: 8212d8fe-4bba-4599-939a-ca1ee3facc48 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:13:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-f9fec2 namespace: kserve-ci-e2e-test resourceVersion: "19402" uid: cc8bb124-2250-43e9-8b31-91242f5f195a kind: Event lastTimestamp: "2026-04-16T15:13:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-f9fec2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:13:45Z" name: isvc-secondary-f9fec2.18a6df1e75254716 namespace: kserve-ci-e2e-test resourceVersion: "19434" uid: 1db833ef-ac8d-4bfd-8731-1673af34fc26 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:04:58.829287Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78 namespace: kserve-ci-e2e-test resourceVersion: "14864" uid: 698398ad-669d-4a53-b8ce-c9e4d415cd4a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78 to ip-10-0-142-46.ec2.internal metadata: creationTimestamp: "2026-04-16T15:04:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:04:58Z" name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78.18a6dea3f5cf86e0 namespace: kserve-ci-e2e-test resourceVersion: "14875" uid: d508e57f-9453-4ea3-9826-ec8527cc8eb1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-794cffb559-pkg6f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:04:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78 namespace: kserve-ci-e2e-test resourceVersion: "14881" uid: 698398ad-669d-4a53-b8ce-c9e4d415cd4a kind: Event lastTimestamp: "2026-04-16T15:04:59Z" message: Add eth0 [10.133.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:04:59Z" name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78.18a6dea40f3dc749 namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 65372c2d-24c2-4a46-9037-cdcd1b935554 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:04:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78 namespace: kserve-ci-e2e-test resourceVersion: "14867" uid: 698398ad-669d-4a53-b8ce-c9e4d415cd4a kind: Event lastTimestamp: "2026-04-16T15:04:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:c49c84e69ddbb36db782821d4aa3d203f458a00c0c04dd506eb662aae339bb8a" already present on machine metadata: creationTimestamp: "2026-04-16T15:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:04:59Z" name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78.18a6dea410756029 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: a8514687-e427-40da-885e-18fb6d686613 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:04:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78 namespace: kserve-ci-e2e-test resourceVersion: "14867" uid: 698398ad-669d-4a53-b8ce-c9e4d415cd4a kind: Event lastTimestamp: "2026-04-16T15:04:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T15:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:04:59Z" name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78.18a6dea4136d31cd namespace: kserve-ci-e2e-test resourceVersion: "14912" uid: f99691ea-ea6f-4fdb-a54b-f752215f830b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:04:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78 namespace: kserve-ci-e2e-test resourceVersion: "14867" uid: 698398ad-669d-4a53-b8ce-c9e4d415cd4a kind: Event lastTimestamp: "2026-04-16T15:04:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T15:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:04:59Z" name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78.18a6dea4141a36ba namespace: kserve-ci-e2e-test resourceVersion: "14915" uid: 36b98d4c-1d91-4965-a0f4-333117fdba49 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78 namespace: kserve-ci-e2e-test resourceVersion: "14867" uid: 698398ad-669d-4a53-b8ce-c9e4d415cd4a kind: Event lastTimestamp: "2026-04-16T15:05:03Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:05:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:03Z" name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78.18a6dea511712b9d namespace: kserve-ci-e2e-test resourceVersion: "14969" uid: 116bb897-ba95-4e09-aa46-2424bf666eb0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78 namespace: kserve-ci-e2e-test resourceVersion: "14867" uid: 698398ad-669d-4a53-b8ce-c9e4d415cd4a kind: Event lastTimestamp: "2026-04-16T15:05:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:05:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:03Z" name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78.18a6dea5148bbe37 namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: ba9b55d1-adb1-4c3f-8bab-7471ee7e7b14 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78 namespace: kserve-ci-e2e-test resourceVersion: "14867" uid: 698398ad-669d-4a53-b8ce-c9e4d415cd4a kind: Event lastTimestamp: "2026-04-16T15:05:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:05:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:03Z" name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78.18a6dea514effd4e namespace: kserve-ci-e2e-test resourceVersion: "14975" uid: 66768d8f-894e-42f0-a435-72c5d48183d2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T15:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78 namespace: kserve-ci-e2e-test resourceVersion: "14867" uid: 698398ad-669d-4a53-b8ce-c9e4d415cd4a kind: Event lastTimestamp: "2026-04-16T15:06:08Z" message: 'Readiness probe failed: dial tcp 10.133.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:08Z" name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78.18a6dea54d8a4dd8 namespace: kserve-ci-e2e-test resourceVersion: "15469" uid: 28db06c1-31fb-42da-a6ee-7b428d18610f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78 namespace: kserve-ci-e2e-test resourceVersion: "14867" uid: 698398ad-669d-4a53-b8ce-c9e4d415cd4a kind: Event lastTimestamp: "2026-04-16T15:06:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:39Z" name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78.18a6debb49a825e8 namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 06d098c5-68c3-4f32-b5b7-1736b8975f0b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:04:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: 1da09225-3639-430c-adae-f68cbf60699a kind: Event lastTimestamp: "2026-04-16T15:04:58Z" message: 'Created pod: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d-jdt78' metadata: creationTimestamp: "2026-04-16T15:04:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:04:58Z" name: isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d.18a6dea3f576a965 namespace: kserve-ci-e2e-test resourceVersion: "14869" uid: 553f3085-34b4-4fbd-9391-9dd418702e48 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:04:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-0fa3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: bf6027cc-1007-410c-9b5e-efd67301078e kind: Event lastTimestamp: "2026-04-16T15:04:58Z" message: Scaled up replica set isvc-sklearn-graph-raw-0fa3d-predictor-5ccf48bd8d from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:04:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:04:58Z" name: isvc-sklearn-graph-raw-0fa3d-predictor.18a6dea3efd72681 namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 2d3d99ad-9418-4072-aefd-67837a6ce943 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-0fa3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14842" uid: 34a127d6-a735-4fd2-8998-bbbb2095a6a2 kind: Event lastTimestamp: "2026-04-16T15:05:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:05:13Z" name: isvc-sklearn-graph-raw-0fa3d-predictor.18a6dea76ed3a8b9 namespace: kserve-ci-e2e-test resourceVersion: "15055" uid: 524a58a2-95fc-4319-af18-a80f171e80a4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-0fa3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14842" uid: 34a127d6-a735-4fd2-8998-bbbb2095a6a2 kind: Event lastTimestamp: "2026-04-16T15:05:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:05:13Z" name: isvc-sklearn-graph-raw-0fa3d-predictor.18a6dea76f74ab89 namespace: kserve-ci-e2e-test resourceVersion: "15056" uid: 6b66b972-d9f2-4fcc-8984-cd8b8ab0c785 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:05:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-0fa3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: 34a127d6-a735-4fd2-8998-bbbb2095a6a2 kind: Event lastTimestamp: "2026-04-16T15:06:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:28Z" name: isvc-sklearn-graph-raw-0fa3d-predictor.18a6deaaee1e7fa9 namespace: kserve-ci-e2e-test resourceVersion: "15651" uid: 71442212-97fa-43be-9e5d-092d8ee197be reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:05:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-0fa3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: 34a127d6-a735-4fd2-8998-bbbb2095a6a2 kind: Event lastTimestamp: "2026-04-16T15:06:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:28Z" name: isvc-sklearn-graph-raw-0fa3d-predictor.18a6deaaee901153 namespace: kserve-ci-e2e-test resourceVersion: "15653" uid: 54b4eabc-798e-4275-a8b9-503dda2edb96 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T15:04:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-0fa3d namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: 8689b753-bb4d-495f-b2b3-1834bf92ce3c kind: Event lastTimestamp: "2026-04-16T15:06:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:04:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:06:18Z" name: isvc-sklearn-graph-raw-0fa3d.18a6dea3efb52427 namespace: kserve-ci-e2e-test resourceVersion: "15552" uid: bf2cdb20-ebb6-4070-bf02-70a2b05c3879 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:04:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-0fa3d namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: 8689b753-bb4d-495f-b2b3-1834bf92ce3c kind: Event lastTimestamp: "2026-04-16T15:04:58Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-0fa3d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-0fa3d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:04:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:04:58Z" name: isvc-sklearn-graph-raw-0fa3d.18a6dea3f109f880 namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: a5919b26-1917-4b97-b872-d698e8739aa8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-0fa3d namespace: kserve-ci-e2e-test resourceVersion: "15549" uid: 8689b753-bb4d-495f-b2b3-1834bf92ce3c kind: Event lastTimestamp: "2026-04-16T15:06:18Z" message: InferenceService [isvc-sklearn-graph-raw-0fa3d] is Ready metadata: creationTimestamp: "2026-04-16T15:06:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:06:18Z" name: isvc-sklearn-graph-raw-0fa3d.18a6deb67bbb5311 namespace: kserve-ci-e2e-test resourceVersion: "15550" uid: 4115d26c-4bb3-492f-80da-cd3eb3e8130b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:06:39.055238Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz namespace: kserve-ci-e2e-test resourceVersion: "15815" uid: 7e700bf1-641c-42b2-92f0-b5a30ba68cf8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz to ip-10-0-142-46.ec2.internal metadata: creationTimestamp: "2026-04-16T15:06:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:06:39Z" name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz.18a6debb4bbe4552 namespace: kserve-ci-e2e-test resourceVersion: "15823" uid: 370aecc5-722d-479f-b125-bb5030dfc800 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-794cffb559-pkg6f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz namespace: kserve-ci-e2e-test resourceVersion: "15828" uid: 7e700bf1-641c-42b2-92f0-b5a30ba68cf8 kind: Event lastTimestamp: "2026-04-16T15:06:39Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:06:39Z" name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz.18a6debb6583ee75 namespace: kserve-ci-e2e-test resourceVersion: "15871" uid: a098c671-7802-477e-91dc-353ad417dc19 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: 7e700bf1-641c-42b2-92f0-b5a30ba68cf8 kind: Event lastTimestamp: "2026-04-16T15:06:39Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:c49c84e69ddbb36db782821d4aa3d203f458a00c0c04dd506eb662aae339bb8a" already present on machine metadata: creationTimestamp: "2026-04-16T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:39Z" name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz.18a6debb66d3c221 namespace: kserve-ci-e2e-test resourceVersion: "15873" uid: d0a38b57-3593-432a-a708-e9d9a5799917 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: 7e700bf1-641c-42b2-92f0-b5a30ba68cf8 kind: Event lastTimestamp: "2026-04-16T15:06:39Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:39Z" name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz.18a6debb69c06a65 namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: d4cc6935-81b9-415b-9427-0bef00c7a2c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: 7e700bf1-641c-42b2-92f0-b5a30ba68cf8 kind: Event lastTimestamp: "2026-04-16T15:06:39Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:39Z" name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz.18a6debb6a390868 namespace: kserve-ci-e2e-test resourceVersion: "15879" uid: 57fd3d99-4c07-4f43-a2ac-944b332cd496 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: 7e700bf1-641c-42b2-92f0-b5a30ba68cf8 kind: Event lastTimestamp: "2026-04-16T15:06:43Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:43Z" name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz.18a6debc6c93781f namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: 38942cd3-21ec-474f-9067-b4da4f6c1359 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: 7e700bf1-641c-42b2-92f0-b5a30ba68cf8 kind: Event lastTimestamp: "2026-04-16T15:06:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:43Z" name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz.18a6debc70cb741f namespace: kserve-ci-e2e-test resourceVersion: "15943" uid: bb0618b8-4c9d-4aad-8346-edd227510e10 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: 7e700bf1-641c-42b2-92f0-b5a30ba68cf8 kind: Event lastTimestamp: "2026-04-16T15:06:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:43Z" name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz.18a6debc71459953 namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: 0d408140-1462-4e38-888c-1fe48857f41f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T15:06:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: 7e700bf1-641c-42b2-92f0-b5a30ba68cf8 kind: Event lastTimestamp: "2026-04-16T15:07:45Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:45Z" name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz.18a6debca89f9934 namespace: kserve-ci-e2e-test resourceVersion: "16405" uid: d12b8ff6-2b9f-40c1-817c-78c41393135d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: 7e700bf1-641c-42b2-92f0-b5a30ba68cf8 kind: Event lastTimestamp: "2026-04-16T15:08:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:08:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:19Z" name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz.18a6ded2a387c4d1 namespace: kserve-ci-e2e-test resourceVersion: "16762" uid: a0f55a51-c67b-4b25-b4ae-da3aa5e2b153 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd namespace: kserve-ci-e2e-test resourceVersion: "15779" uid: 314c5980-c1e5-4ad5-833c-6cf122fc5010 kind: Event lastTimestamp: "2026-04-16T15:06:39Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd-7hkkz' metadata: creationTimestamp: "2026-04-16T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:39Z" name: isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd.18a6debb4b582468 namespace: kserve-ci-e2e-test resourceVersion: "15817" uid: 23b0cfb7-140f-47c5-a38c-c60a31ee1b0e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-007f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: 3c6d2b7e-6606-4e83-a26f-638b769df54a kind: Event lastTimestamp: "2026-04-16T15:06:38Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-007f8-predictor-578bc4d7bd from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:38Z" name: isvc-sklearn-graph-raw-hpa-007f8-predictor.18a6debb46bc921d namespace: kserve-ci-e2e-test resourceVersion: "15782" uid: 006a342a-4a60-41c8-adff-b87bea194890 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:06:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-007f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15778" uid: 402ace11-3dae-42b4-b282-04c584e1d506 kind: Event lastTimestamp: "2026-04-16T15:07:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:07:09Z" name: isvc-sklearn-graph-raw-hpa-007f8-predictor.18a6debec5b9dec3 namespace: kserve-ci-e2e-test resourceVersion: "16140" uid: 97a46643-bdf2-4ad3-b02c-9a53e9aff4fe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:06:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-007f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15778" uid: 402ace11-3dae-42b4-b282-04c584e1d506 kind: Event lastTimestamp: "2026-04-16T15:07:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:07:09Z" name: isvc-sklearn-graph-raw-hpa-007f8-predictor.18a6debec61b89f8 namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: 8d24392d-41f8-444c-8b48-3beec62cfb46 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T15:07:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-007f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: 402ace11-3dae-42b4-b282-04c584e1d506 kind: Event lastTimestamp: "2026-04-16T15:08:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:07:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:09Z" name: isvc-sklearn-graph-raw-hpa-007f8-predictor.18a6dec5c3e421c6 namespace: kserve-ci-e2e-test resourceVersion: "16601" uid: 404d4c7d-eaa6-46f4-9eca-49f8ba0a2cdb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T15:07:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-007f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: 402ace11-3dae-42b4-b282-04c584e1d506 kind: Event lastTimestamp: "2026-04-16T15:08:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:07:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:09Z" name: isvc-sklearn-graph-raw-hpa-007f8-predictor.18a6dec5c44786cc namespace: kserve-ci-e2e-test resourceVersion: "16604" uid: 2d4ade39-0578-4408-8e1a-ea96a69497eb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T15:06:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-007f8 namespace: kserve-ci-e2e-test resourceVersion: "15769" uid: 2b4f51df-b578-4789-ae46-160bba6cb512 kind: Event lastTimestamp: "2026-04-16T15:07:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:07:55Z" name: isvc-sklearn-graph-raw-hpa-007f8.18a6debb469c1d1a namespace: kserve-ci-e2e-test resourceVersion: "16496" uid: b196cdc7-2c2b-4a19-84fd-d890ef049f89 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-007f8 namespace: kserve-ci-e2e-test resourceVersion: "16494" uid: 2b4f51df-b578-4789-ae46-160bba6cb512 kind: Event lastTimestamp: "2026-04-16T15:07:55Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-007f8] is Ready metadata: creationTimestamp: "2026-04-16T15:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:07:55Z" name: isvc-sklearn-graph-raw-hpa-007f8.18a6decd34941771 namespace: kserve-ci-e2e-test resourceVersion: "16495" uid: aa2a7156-c2b3-433f-8302-38bb52aa9078 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:09:54.604053Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25 namespace: kserve-ci-e2e-test resourceVersion: "17619" uid: 0dc64f04-3d14-419a-8df4-dd29eb39b88d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25 to ip-10-0-142-46.ec2.internal metadata: creationTimestamp: "2026-04-16T15:09:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:09:54Z" name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25.18a6dee8d35c440d namespace: kserve-ci-e2e-test resourceVersion: "17626" uid: 01e9ec4e-3d85-4f78-b217-4e3ffc411784 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-794cffb559-pkg6f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:09:55Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25 namespace: kserve-ci-e2e-test resourceVersion: "17629" uid: 0dc64f04-3d14-419a-8df4-dd29eb39b88d kind: Event lastTimestamp: "2026-04-16T15:09:55Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:09:55Z" name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25.18a6dee8ec750b00 namespace: kserve-ci-e2e-test resourceVersion: "17653" uid: fca7c690-1b9f-4bb5-a02b-96885fb6f11e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25 namespace: kserve-ci-e2e-test resourceVersion: "17621" uid: 0dc64f04-3d14-419a-8df4-dd29eb39b88d kind: Event lastTimestamp: "2026-04-16T15:09:55Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:c49c84e69ddbb36db782821d4aa3d203f458a00c0c04dd506eb662aae339bb8a" already present on machine metadata: creationTimestamp: "2026-04-16T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:09:55Z" name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25.18a6dee8ed90705a namespace: kserve-ci-e2e-test resourceVersion: "17655" uid: 1be0247c-7a0b-4614-8680-8d2a01dd3ec5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25 namespace: kserve-ci-e2e-test resourceVersion: "17621" uid: 0dc64f04-3d14-419a-8df4-dd29eb39b88d kind: Event lastTimestamp: "2026-04-16T15:09:55Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:09:55Z" name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25.18a6dee8f02dafd8 namespace: kserve-ci-e2e-test resourceVersion: "17657" uid: da5806aa-3277-4d05-9bff-03164379c150 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25 namespace: kserve-ci-e2e-test resourceVersion: "17621" uid: 0dc64f04-3d14-419a-8df4-dd29eb39b88d kind: Event lastTimestamp: "2026-04-16T15:09:55Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:09:55Z" name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25.18a6dee8f0d984df namespace: kserve-ci-e2e-test resourceVersion: "17658" uid: 09fe4695-6eb7-4bfb-b7bb-59d46733a995 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:09:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25 namespace: kserve-ci-e2e-test resourceVersion: "17621" uid: 0dc64f04-3d14-419a-8df4-dd29eb39b88d kind: Event lastTimestamp: "2026-04-16T15:09:59Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:09:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:09:59Z" name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25.18a6dee9f94ae0e6 namespace: kserve-ci-e2e-test resourceVersion: "17701" uid: 733f1e18-7479-4d71-9ca0-db555bca452c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:09:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25 namespace: kserve-ci-e2e-test resourceVersion: "17621" uid: 0dc64f04-3d14-419a-8df4-dd29eb39b88d kind: Event lastTimestamp: "2026-04-16T15:09:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:09:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:09:59Z" name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25.18a6dee9fbefcdb6 namespace: kserve-ci-e2e-test resourceVersion: "17705" uid: c684f815-cf33-41f0-9a71-79745270ad6b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:09:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25 namespace: kserve-ci-e2e-test resourceVersion: "17621" uid: 0dc64f04-3d14-419a-8df4-dd29eb39b88d kind: Event lastTimestamp: "2026-04-16T15:09:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:09:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:09:59Z" name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25.18a6dee9fc580586 namespace: kserve-ci-e2e-test resourceVersion: "17706" uid: 53100a00-58f9-4602-bf7e-e03294abff56 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 17 eventTime: null firstTimestamp: "2026-04-16T15:10:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25 namespace: kserve-ci-e2e-test resourceVersion: "17621" uid: 0dc64f04-3d14-419a-8df4-dd29eb39b88d kind: Event lastTimestamp: "2026-04-16T15:12:32Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:12:32Z" name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25.18a6deea3531a2ed namespace: kserve-ci-e2e-test resourceVersion: "18876" uid: e87b7b38-6168-45f6-a4a9-63cf3d164fee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:12:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25 namespace: kserve-ci-e2e-test resourceVersion: "17621" uid: 0dc64f04-3d14-419a-8df4-dd29eb39b88d kind: Event lastTimestamp: "2026-04-16T15:12:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:12:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:12:24Z" name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25.18a6df0bcbd2423b namespace: kserve-ci-e2e-test resourceVersion: "18783" uid: 42037462-4baa-47c3-b6b3-e9ece7ab96fa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:09:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454 namespace: kserve-ci-e2e-test resourceVersion: "17600" uid: db7ef46e-c4c6-43ef-9e73-229c17d290b5 kind: Event lastTimestamp: "2026-04-16T15:09:54Z" message: 'Created pod: isvc-sklearn-scale-raw-ac40b-predictor-56648db454-vwl25' metadata: creationTimestamp: "2026-04-16T15:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:09:54Z" name: isvc-sklearn-scale-raw-ac40b-predictor-56648db454.18a6dee8d2ef4ad3 namespace: kserve-ci-e2e-test resourceVersion: "17620" uid: 74bb4382-31ad-448d-9372-d33b21eb9e8a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:09:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-ac40b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17595" uid: 8fb2fa1f-33f6-4694-85e4-a29aa5cb1b6d kind: Event lastTimestamp: "2026-04-16T15:09:54Z" message: Scaled up replica set isvc-sklearn-scale-raw-ac40b-predictor-56648db454 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:09:54Z" name: isvc-sklearn-scale-raw-ac40b-predictor.18a6dee8d049fccb namespace: kserve-ci-e2e-test resourceVersion: "17605" uid: e4572bde-cd17-4a27-ac45-87ac2e86096f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:10:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-ac40b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17599" uid: 8c95a593-53e8-480b-9110-d603724335a2 kind: Event lastTimestamp: "2026-04-16T15:10:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:10:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:10:09Z" name: isvc-sklearn-scale-raw-ac40b-predictor.18a6deec4f3ee3e7 namespace: kserve-ci-e2e-test resourceVersion: "17784" uid: f5f37545-8b71-4efd-824c-5793f8fd678b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:10:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-ac40b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17599" uid: 8c95a593-53e8-480b-9110-d603724335a2 kind: Event lastTimestamp: "2026-04-16T15:10:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:10:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:10:09Z" name: isvc-sklearn-scale-raw-ac40b-predictor.18a6deec4fa7248b namespace: kserve-ci-e2e-test resourceVersion: "17786" uid: c8244ad4-6a56-4a1a-8fb4-55806febd514 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T15:10:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-ac40b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17785" uid: 8c95a593-53e8-480b-9110-d603724335a2 kind: Event lastTimestamp: "2026-04-16T15:12:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:12:24Z" name: isvc-sklearn-scale-raw-ac40b-predictor.18a6deefce40bfeb namespace: kserve-ci-e2e-test resourceVersion: "18752" uid: 483cac23-7ab1-4f9f-b848-9cff0dab5bac reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T15:10:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-ac40b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17785" uid: 8c95a593-53e8-480b-9110-d603724335a2 kind: Event lastTimestamp: "2026-04-16T15:12:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:12:24Z" name: isvc-sklearn-scale-raw-ac40b-predictor.18a6deefcea61744 namespace: kserve-ci-e2e-test resourceVersion: "18753" uid: 2ebfc6d2-ffcf-4674-a813-29dfbb0f02af reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T15:09:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-ac40b namespace: kserve-ci-e2e-test resourceVersion: "17582" uid: e39a0b45-ffb3-46f4-b376-837417d3c7b2 kind: Event lastTimestamp: "2026-04-16T15:12:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:12:22Z" name: isvc-sklearn-scale-raw-ac40b.18a6dee8d0524b78 namespace: kserve-ci-e2e-test resourceVersion: "18733" uid: 13fa88ab-4069-48f0-8f9a-01a565a7067d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:09:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-ac40b namespace: kserve-ci-e2e-test resourceVersion: "17582" uid: e39a0b45-ffb3-46f4-b376-837417d3c7b2 kind: Event lastTimestamp: "2026-04-16T15:09:54Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-ac40b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-ac40b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:09:54Z" name: isvc-sklearn-scale-raw-ac40b.18a6dee8d10186df namespace: kserve-ci-e2e-test resourceVersion: "17612" uid: 8436f862-1790-40fd-bed0-7057fe029daf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:12:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-ac40b namespace: kserve-ci-e2e-test resourceVersion: "18731" uid: e39a0b45-ffb3-46f4-b376-837417d3c7b2 kind: Event lastTimestamp: "2026-04-16T15:12:22Z" message: InferenceService [isvc-sklearn-scale-raw-ac40b] is Ready metadata: creationTimestamp: "2026-04-16T15:12:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:12:22Z" name: isvc-sklearn-scale-raw-ac40b.18a6df0b3cc7916d namespace: kserve-ci-e2e-test resourceVersion: "18732" uid: 293ab390-974a-4c82-aa33-e702de0db167 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:04:58.872110Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k namespace: kserve-ci-e2e-test resourceVersion: "14882" uid: 01e75b68-a5d7-41a3-b185-020a24ed1293 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k to ip-10-0-142-46.ec2.internal metadata: creationTimestamp: "2026-04-16T15:04:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:04:58Z" name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k.18a6dea3f85cf876 namespace: kserve-ci-e2e-test resourceVersion: "14889" uid: 44143fb6-038c-452d-84a3-6f430f8099ea reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-794cffb559-pkg6f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:04:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k namespace: kserve-ci-e2e-test resourceVersion: "14897" uid: 01e75b68-a5d7-41a3-b185-020a24ed1293 kind: Event lastTimestamp: "2026-04-16T15:04:59Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:04:59Z" name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k.18a6dea412630ac7 namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: 1f0b8ae5-10b8-4727-8b21-ca6d2a8fb431 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:04:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: 01e75b68-a5d7-41a3-b185-020a24ed1293 kind: Event lastTimestamp: "2026-04-16T15:04:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:c49c84e69ddbb36db782821d4aa3d203f458a00c0c04dd506eb662aae339bb8a" already present on machine metadata: creationTimestamp: "2026-04-16T15:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:04:59Z" name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k.18a6dea4139d8260 namespace: kserve-ci-e2e-test resourceVersion: "14913" uid: 1e6ff493-91c3-4434-9a9a-eb33d88d612e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:04:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: 01e75b68-a5d7-41a3-b185-020a24ed1293 kind: Event lastTimestamp: "2026-04-16T15:04:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T15:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:04:59Z" name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k.18a6dea4168c7718 namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: d33d0549-8d4f-4c61-a85d-285c9aab3891 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:04:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: 01e75b68-a5d7-41a3-b185-020a24ed1293 kind: Event lastTimestamp: "2026-04-16T15:04:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T15:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:04:59Z" name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k.18a6dea416fc9b8c namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: 759c46a9-7b2c-4920-a0d3-9b1130d48082 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: 01e75b68-a5d7-41a3-b185-020a24ed1293 kind: Event lastTimestamp: "2026-04-16T15:05:03Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-16T15:05:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:03Z" name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k.18a6dea5115ce316 namespace: kserve-ci-e2e-test resourceVersion: "14968" uid: 1898cae2-c42b-41e7-889d-3b4a83699d91 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: 01e75b68-a5d7-41a3-b185-020a24ed1293 kind: Event lastTimestamp: "2026-04-16T15:05:25Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 21.719s (21.719s including waiting). Image size: 1306229499 bytes.' metadata: creationTimestamp: "2026-04-16T15:05:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:25Z" name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k.18a6deaa1fed9ec4 namespace: kserve-ci-e2e-test resourceVersion: "15144" uid: 2e34a573-433b-4755-a9c2-4fcb0574fc6e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: 01e75b68-a5d7-41a3-b185-020a24ed1293 kind: Event lastTimestamp: "2026-04-16T15:05:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:05:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:25Z" name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k.18a6deaa232015bd namespace: kserve-ci-e2e-test resourceVersion: "15146" uid: c09c9322-0fc2-4f03-bcdd-0692d7f42bf4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: 01e75b68-a5d7-41a3-b185-020a24ed1293 kind: Event lastTimestamp: "2026-04-16T15:05:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:05:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:25Z" name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k.18a6deaa23c9f068 namespace: kserve-ci-e2e-test resourceVersion: "15147" uid: d835218c-075b-451a-b90c-709214f35e12 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T15:05:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: 01e75b68-a5d7-41a3-b185-020a24ed1293 kind: Event lastTimestamp: "2026-04-16T15:06:16Z" message: 'Readiness probe failed: dial tcp 10.133.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:05:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:16Z" name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k.18a6deaa354eb7b5 namespace: kserve-ci-e2e-test resourceVersion: "15530" uid: fc17fe50-b70f-499c-961a-972e4784e713 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: 01e75b68-a5d7-41a3-b185-020a24ed1293 kind: Event lastTimestamp: "2026-04-16T15:06:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:39Z" name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k.18a6debb545ead0c namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: 11194738-d789-40a9-8fb4-41129f225de8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:04:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95 namespace: kserve-ci-e2e-test resourceVersion: "14863" uid: 660223cf-9d9d-486f-8fbf-72d4368a5314 kind: Event lastTimestamp: "2026-04-16T15:04:58Z" message: 'Created pod: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95-z2d7k' metadata: creationTimestamp: "2026-04-16T15:04:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:04:58Z" name: isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95.18a6dea3f800aa7e namespace: kserve-ci-e2e-test resourceVersion: "14887" uid: 1f476bce-3af2-4a18-9296-f5b420faa128 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:04:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-0fa3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14861" uid: 27343eb6-8754-4bb8-979e-6dce209ef922 kind: Event lastTimestamp: "2026-04-16T15:04:58Z" message: Scaled up replica set isvc-xgboost-graph-raw-0fa3d-predictor-74f94bbd95 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:04:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:04:58Z" name: isvc-xgboost-graph-raw-0fa3d-predictor.18a6dea3f570e63e namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 5f729937-178c-481c-a16b-72b8f3accd40 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:05:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-0fa3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14862" uid: a5d6bca4-74fb-49d1-a1b8-d1e2dcfa7f6a kind: Event lastTimestamp: "2026-04-16T15:05:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:05:43Z" name: isvc-xgboost-graph-raw-0fa3d-predictor.18a6dea773d531be namespace: kserve-ci-e2e-test resourceVersion: "15291" uid: 91603d26-a3fb-4f9e-975a-d5990a8688ba reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:05:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-0fa3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14862" uid: a5d6bca4-74fb-49d1-a1b8-d1e2dcfa7f6a kind: Event lastTimestamp: "2026-04-16T15:05:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:05:43Z" name: isvc-xgboost-graph-raw-0fa3d-predictor.18a6dea7743682c0 namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: 503e3c28-2d1f-42bb-9042-93cdbbdb8e2c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:05:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-0fa3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15058" uid: a5d6bca4-74fb-49d1-a1b8-d1e2dcfa7f6a kind: Event lastTimestamp: "2026-04-16T15:06:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:28Z" name: isvc-xgboost-graph-raw-0fa3d-predictor.18a6deb1f06b3cc6 namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: a6f353ff-16fb-4238-9c21-b1af4353698c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:05:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-0fa3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15058" uid: a5d6bca4-74fb-49d1-a1b8-d1e2dcfa7f6a kind: Event lastTimestamp: "2026-04-16T15:06:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:28Z" name: isvc-xgboost-graph-raw-0fa3d-predictor.18a6deb1f0c89d17 namespace: kserve-ci-e2e-test resourceVersion: "15660" uid: b50f2858-af54-424f-b46b-2ca09e3f0aa7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T15:04:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-0fa3d namespace: kserve-ci-e2e-test resourceVersion: "14851" uid: 6bf951c0-4146-4be6-8969-d7288641df99 kind: Event lastTimestamp: "2026-04-16T15:06:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:04:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:06:26Z" name: isvc-xgboost-graph-raw-0fa3d.18a6dea3f540628a namespace: kserve-ci-e2e-test resourceVersion: "15613" uid: 7109a030-28ff-4404-b56d-4ceee9939251 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:04:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-0fa3d namespace: kserve-ci-e2e-test resourceVersion: "14851" uid: 6bf951c0-4146-4be6-8969-d7288641df99 kind: Event lastTimestamp: "2026-04-16T15:04:58Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-0fa3d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-0fa3d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:04:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:04:58Z" name: isvc-xgboost-graph-raw-0fa3d.18a6dea3f5acb6aa namespace: kserve-ci-e2e-test resourceVersion: "14872" uid: 304d3a67-e047-4cf1-baa4-8e62386783a6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-0fa3d namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: 6bf951c0-4146-4be6-8969-d7288641df99 kind: Event lastTimestamp: "2026-04-16T15:06:26Z" message: InferenceService [isvc-xgboost-graph-raw-0fa3d] is Ready metadata: creationTimestamp: "2026-04-16T15:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:06:26Z" name: isvc-xgboost-graph-raw-0fa3d.18a6deb86d0b4738 namespace: kserve-ci-e2e-test resourceVersion: "15612" uid: 2ecfe4b8-5a28-4762-9002-428df58c4eb8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:06:39.102688Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k namespace: kserve-ci-e2e-test resourceVersion: "15830" uid: 8db24c2d-d0c3-4ebf-8d45-b9f0103a573c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k to ip-10-0-142-46.ec2.internal metadata: creationTimestamp: "2026-04-16T15:06:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:06:39Z" name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k.18a6debb4e927fc2 namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: 60234fd2-6dec-42ee-ba2f-c27faafa606c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-794cffb559-pkg6f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 8db24c2d-d0c3-4ebf-8d45-b9f0103a573c kind: Event lastTimestamp: "2026-04-16T15:06:39Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:06:39Z" name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k.18a6debb688b49b5 namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: f864c99e-8346-4e67-bc7f-a57561e90743 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 8db24c2d-d0c3-4ebf-8d45-b9f0103a573c kind: Event lastTimestamp: "2026-04-16T15:06:39Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:c49c84e69ddbb36db782821d4aa3d203f458a00c0c04dd506eb662aae339bb8a" already present on machine metadata: creationTimestamp: "2026-04-16T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:39Z" name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k.18a6debb69b134fd namespace: kserve-ci-e2e-test resourceVersion: "15876" uid: 3a53bf6d-b797-4a0f-a957-41ee5ddadbce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 8db24c2d-d0c3-4ebf-8d45-b9f0103a573c kind: Event lastTimestamp: "2026-04-16T15:06:39Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:39Z" name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k.18a6debb6c481607 namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: d236973a-ab22-4409-aac3-4ec172e67c45 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 8db24c2d-d0c3-4ebf-8d45-b9f0103a573c kind: Event lastTimestamp: "2026-04-16T15:06:39Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:39Z" name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k.18a6debb6d3311a3 namespace: kserve-ci-e2e-test resourceVersion: "15881" uid: eaf80d48-0ccf-4d7d-ada6-63494a16ab1b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 8db24c2d-d0c3-4ebf-8d45-b9f0103a573c kind: Event lastTimestamp: "2026-04-16T15:06:43Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-16T15:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:43Z" name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k.18a6debc6ca97c52 namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: 061aef74-b093-46f4-a1bc-feaeb618fdca reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 8db24c2d-d0c3-4ebf-8d45-b9f0103a573c kind: Event lastTimestamp: "2026-04-16T15:06:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:43Z" name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k.18a6debc70bf456c namespace: kserve-ci-e2e-test resourceVersion: "15941" uid: 5f1cadc2-363a-4f80-baa2-76ca7b1d1839 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 8db24c2d-d0c3-4ebf-8d45-b9f0103a573c kind: Event lastTimestamp: "2026-04-16T15:06:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:43Z" name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k.18a6debc713884ed namespace: kserve-ci-e2e-test resourceVersion: "15944" uid: d11ea730-bc39-4d48-9113-dbc9fdc38762 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T15:06:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 8db24c2d-d0c3-4ebf-8d45-b9f0103a573c kind: Event lastTimestamp: "2026-04-16T15:07:35Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:35Z" name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k.18a6debca8ba37bf namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: 6fee5f7d-7266-4316-bacc-60be5b0b34be reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 8db24c2d-d0c3-4ebf-8d45-b9f0103a573c kind: Event lastTimestamp: "2026-04-16T15:08:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:08:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:19Z" name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k.18a6ded2a90594a5 namespace: kserve-ci-e2e-test resourceVersion: "16794" uid: 2a1ee9fe-0a97-4b6b-8d67-8361f8006da2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67 namespace: kserve-ci-e2e-test resourceVersion: "15814" uid: 6058f191-c3f2-4ce8-a9b8-4f86bbcecf3f kind: Event lastTimestamp: "2026-04-16T15:06:39Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67-lhs6k' metadata: creationTimestamp: "2026-04-16T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:39Z" name: isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67.18a6debb4e1a3262 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: c089fa38-4f7d-4c7a-bbec-1c46566af356 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-007f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: f4d9cb2e-63f6-4796-bfb4-f47189517303 kind: Event lastTimestamp: "2026-04-16T15:06:39Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-007f8-predictor-775bddbb67 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:39Z" name: isvc-xgboost-graph-raw-hpa-007f8-predictor.18a6debb4b54b552 namespace: kserve-ci-e2e-test resourceVersion: "15818" uid: 6ebe7af8-c874-46b7-acb9-93319721e209 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-007f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15813" uid: 8c78794b-269f-4598-81eb-834f4ea144c6 kind: Event lastTimestamp: "2026-04-16T15:06:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:54Z" name: isvc-xgboost-graph-raw-hpa-007f8-predictor.18a6debec9bcb52a namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: 53f779e3-aa72-4e59-9533-161a1d09fa04 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-007f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15813" uid: 8c78794b-269f-4598-81eb-834f4ea144c6 kind: Event lastTimestamp: "2026-04-16T15:06:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:54Z" name: isvc-xgboost-graph-raw-hpa-007f8-predictor.18a6debeca193cb7 namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 010f4c64-ff4e-4e51-87a9-513e9dbedfbb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:07:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-007f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16025" uid: 8c78794b-269f-4598-81eb-834f4ea144c6 kind: Event lastTimestamp: "2026-04-16T15:08:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:09Z" name: isvc-xgboost-graph-raw-hpa-007f8-predictor.18a6dec248a49d45 namespace: kserve-ci-e2e-test resourceVersion: "16613" uid: df76c5de-4e17-4d1e-8d2b-a2c25f611eb3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:07:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-007f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "16025" uid: 8c78794b-269f-4598-81eb-834f4ea144c6 kind: Event lastTimestamp: "2026-04-16T15:08:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:09Z" name: isvc-xgboost-graph-raw-hpa-007f8-predictor.18a6dec24905184a namespace: kserve-ci-e2e-test resourceVersion: "16615" uid: 05d5606a-2080-4f54-ad6a-22dba6c74012 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T15:06:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-007f8 namespace: kserve-ci-e2e-test resourceVersion: "15796" uid: b66b042c-8dfa-4bc1-bbae-d85230050c5e kind: Event lastTimestamp: "2026-04-16T15:07:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:07:45Z" name: isvc-xgboost-graph-raw-hpa-007f8.18a6debb4b306ecd namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: 2557b423-5668-42c0-b49b-d5b720059431 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-007f8 namespace: kserve-ci-e2e-test resourceVersion: "16412" uid: b66b042c-8dfa-4bc1-bbae-d85230050c5e kind: Event lastTimestamp: "2026-04-16T15:07:45Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-007f8] is Ready metadata: creationTimestamp: "2026-04-16T15:07:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:07:45Z" name: isvc-xgboost-graph-raw-hpa-007f8.18a6decae06f8f7f namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: 01cd4678-1649-4851-b04d-31f33519346b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:08:19.327326Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-116da-predictor-6578d49f9-hhb9v namespace: kserve-ci-e2e-test resourceVersion: "16763" uid: dceed32f-55e0-4b72-bdda-061768e385ff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-116da-predictor-6578d49f9-hhb9v to ip-10-0-142-46.ec2.internal metadata: creationTimestamp: "2026-04-16T15:08:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:08:19Z" name: message-dumper-raw-116da-predictor-6578d49f9-hhb9v.18a6ded2a46ccfbe namespace: kserve-ci-e2e-test resourceVersion: "16769" uid: 35701f87-a62f-4192-8033-4602728a63e0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-794cffb559-pkg6f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:19Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-116da-predictor-6578d49f9-hhb9v namespace: kserve-ci-e2e-test resourceVersion: "16775" uid: dceed32f-55e0-4b72-bdda-061768e385ff kind: Event lastTimestamp: "2026-04-16T15:08:19Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:08:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:08:19Z" name: message-dumper-raw-116da-predictor-6578d49f9-hhb9v.18a6ded2ac1791c6 namespace: kserve-ci-e2e-test resourceVersion: "16797" uid: 6f10c606-80d3-4eee-840d-ba702a4a6186 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-116da-predictor-6578d49f9-hhb9v namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: dceed32f-55e0-4b72-bdda-061768e385ff kind: Event lastTimestamp: "2026-04-16T15:08:19Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-16T15:08:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:19Z" name: message-dumper-raw-116da-predictor-6578d49f9-hhb9v.18a6ded2ad392b23 namespace: kserve-ci-e2e-test resourceVersion: "16800" uid: d789c50e-e0a9-4047-8915-4c164e077892 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-116da-predictor-6578d49f9-hhb9v namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: dceed32f-55e0-4b72-bdda-061768e385ff kind: Event lastTimestamp: "2026-04-16T15:08:20Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.127s (1.127s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-16T15:08:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:20Z" name: message-dumper-raw-116da-predictor-6578d49f9-hhb9v.18a6ded2f06bd910 namespace: kserve-ci-e2e-test resourceVersion: "16808" uid: 9d2b90f8-a42a-4d25-8cd8-589f41922c5c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-116da-predictor-6578d49f9-hhb9v namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: dceed32f-55e0-4b72-bdda-061768e385ff kind: Event lastTimestamp: "2026-04-16T15:08:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:08:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:20Z" name: message-dumper-raw-116da-predictor-6578d49f9-hhb9v.18a6ded2f6fed750 namespace: kserve-ci-e2e-test resourceVersion: "16810" uid: b93c3c63-391e-4140-903b-cf69e038f5f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-116da-predictor-6578d49f9-hhb9v namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: dceed32f-55e0-4b72-bdda-061768e385ff kind: Event lastTimestamp: "2026-04-16T15:08:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:08:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:20Z" name: message-dumper-raw-116da-predictor-6578d49f9-hhb9v.18a6ded2f761bbd2 namespace: kserve-ci-e2e-test resourceVersion: "16811" uid: 4358573a-f338-4a01-982a-17213458609a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:09:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-116da-predictor-6578d49f9-hhb9v namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: dceed32f-55e0-4b72-bdda-061768e385ff kind: Event lastTimestamp: "2026-04-16T15:09:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:09:54Z" name: message-dumper-raw-116da-predictor-6578d49f9-hhb9v.18a6dee8d71c86f6 namespace: kserve-ci-e2e-test resourceVersion: "17645" uid: 1256b2e7-a2e9-428e-9da6-197e2e708e01 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-116da-predictor-6578d49f9 namespace: kserve-ci-e2e-test resourceVersion: "16741" uid: 51c2a31b-d2f0-4529-84b3-c02d058225dd kind: Event lastTimestamp: "2026-04-16T15:08:19Z" message: 'Created pod: message-dumper-raw-116da-predictor-6578d49f9-hhb9v' metadata: creationTimestamp: "2026-04-16T15:08:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:19Z" name: message-dumper-raw-116da-predictor-6578d49f9.18a6ded2a40f4f5a namespace: kserve-ci-e2e-test resourceVersion: "16767" uid: ec8b14f6-2a0b-4ce7-b47f-c74faf387055 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-116da-predictor namespace: kserve-ci-e2e-test resourceVersion: "16738" uid: fb034afa-b9eb-482a-8031-af9182c9f26f kind: Event lastTimestamp: "2026-04-16T15:08:19Z" message: Scaled up replica set message-dumper-raw-116da-predictor-6578d49f9 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:08:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:19Z" name: message-dumper-raw-116da-predictor.18a6ded2a0f07156 namespace: kserve-ci-e2e-test resourceVersion: "16744" uid: 864bd99c-52bf-47ad-9d83-e6156d15005f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:08:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-116da-predictor namespace: kserve-ci-e2e-test resourceVersion: "16740" uid: b4cd2d22-f6d6-4388-b809-04e880e788b3 kind: Event lastTimestamp: "2026-04-16T15:09:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:08:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:09:04Z" name: message-dumper-raw-116da-predictor.18a6ded6200d21a9 namespace: kserve-ci-e2e-test resourceVersion: "17213" uid: 63d2f99c-81cd-4fc3-b191-175c34f934bf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:08:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-116da-predictor namespace: kserve-ci-e2e-test resourceVersion: "16740" uid: b4cd2d22-f6d6-4388-b809-04e880e788b3 kind: Event lastTimestamp: "2026-04-16T15:09:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:08:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:09:04Z" name: message-dumper-raw-116da-predictor.18a6ded62042675a namespace: kserve-ci-e2e-test resourceVersion: "17214" uid: 8a1568c8-b974-4596-ba38-3267102fecf1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T15:08:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-116da namespace: kserve-ci-e2e-test resourceVersion: "16732" uid: 32b03315-1c08-49cf-9fe4-d24f4b8f0d85 kind: Event lastTimestamp: "2026-04-16T15:08:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:08:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:08:21Z" name: message-dumper-raw-116da.18a6ded2a0d1804a namespace: kserve-ci-e2e-test resourceVersion: "16828" uid: 3126e8a2-4d91-4b5e-8b20-12efdad9e5d3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-116da namespace: kserve-ci-e2e-test resourceVersion: "16743" uid: 32b03315-1c08-49cf-9fe4-d24f4b8f0d85 kind: Event lastTimestamp: "2026-04-16T15:08:19Z" message: 'Failed to update status for InferenceService "message-dumper-raw-116da": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-116da": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:08:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:08:19Z" name: message-dumper-raw-116da.18a6ded2a3f0049e namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 983d1487-1396-47e1-a227-df60de9d0e7c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-116da namespace: kserve-ci-e2e-test resourceVersion: "16826" uid: 32b03315-1c08-49cf-9fe4-d24f4b8f0d85 kind: Event lastTimestamp: "2026-04-16T15:08:21Z" message: InferenceService [message-dumper-raw-116da] is Ready metadata: creationTimestamp: "2026-04-16T15:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:08:21Z" name: message-dumper-raw-116da.18a6ded31960d3b1 namespace: kserve-ci-e2e-test resourceVersion: "16827" uid: 2a7cd97a-1849-4f13-bfb8-4e15803456a5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:06:28.791331Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-0fa3d-65695c4586-f64qw namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: 55501623-1e85-497c-8e95-44fd6fbfbe0c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-0fa3d-65695c4586-f64qw to ip-10-0-142-46.ec2.internal metadata: creationTimestamp: "2026-04-16T15:06:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:06:28Z" name: model-chainer-raw-0fa3d-65695c4586-f64qw.18a6deb8e7f771cf namespace: kserve-ci-e2e-test resourceVersion: "15646" uid: 2ba773cc-b651-4203-b397-0b8fccb7dd14 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-794cffb559-pkg6f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:28Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-0fa3d-65695c4586-f64qw namespace: kserve-ci-e2e-test resourceVersion: "15643" uid: 55501623-1e85-497c-8e95-44fd6fbfbe0c kind: Event lastTimestamp: "2026-04-16T15:06:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-0fa3d-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T15:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:28Z" name: model-chainer-raw-0fa3d-65695c4586-f64qw.18a6deb8f2313b8b namespace: kserve-ci-e2e-test resourceVersion: "15662" uid: e3dbd1d6-a689-44f0-ad03-c1bf2b8c08a3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:29Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-0fa3d-65695c4586-f64qw namespace: kserve-ci-e2e-test resourceVersion: "15649" uid: 55501623-1e85-497c-8e95-44fd6fbfbe0c kind: Event lastTimestamp: "2026-04-16T15:06:29Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:06:29Z" name: model-chainer-raw-0fa3d-65695c4586-f64qw.18a6deb924bab628 namespace: kserve-ci-e2e-test resourceVersion: "15670" uid: fee91dd0-5f6d-4e42-b977-423151024dae reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-0fa3d} kind: Pod name: model-chainer-raw-0fa3d-65695c4586-f64qw namespace: kserve-ci-e2e-test resourceVersion: "15643" uid: 55501623-1e85-497c-8e95-44fd6fbfbe0c kind: Event lastTimestamp: "2026-04-16T15:06:29Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:daaaf0b7769b899c00520a668e190161eefaa93b6eb75bfb8161ae18d89812c6" metadata: creationTimestamp: "2026-04-16T15:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:29Z" name: model-chainer-raw-0fa3d-65695c4586-f64qw.18a6deb925fe50ed namespace: kserve-ci-e2e-test resourceVersion: "15672" uid: 92d702cd-9e16-466b-8982-c62ca0a6591a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-0fa3d} kind: Pod name: model-chainer-raw-0fa3d-65695c4586-f64qw namespace: kserve-ci-e2e-test resourceVersion: "15643" uid: 55501623-1e85-497c-8e95-44fd6fbfbe0c kind: Event lastTimestamp: "2026-04-16T15:06:32Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:daaaf0b7769b899c00520a668e190161eefaa93b6eb75bfb8161ae18d89812c6" in 2.912s (2.912s including waiting). Image size: 216232057 bytes.' metadata: creationTimestamp: "2026-04-16T15:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:32Z" name: model-chainer-raw-0fa3d-65695c4586-f64qw.18a6deb9d39241ca namespace: kserve-ci-e2e-test resourceVersion: "15694" uid: 159f011e-6319-41ff-b7d2-ce852e5f2505 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-0fa3d} kind: Pod name: model-chainer-raw-0fa3d-65695c4586-f64qw namespace: kserve-ci-e2e-test resourceVersion: "15643" uid: 55501623-1e85-497c-8e95-44fd6fbfbe0c kind: Event lastTimestamp: "2026-04-16T15:06:32Z" message: 'Created container: model-chainer-raw-0fa3d' metadata: creationTimestamp: "2026-04-16T15:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:32Z" name: model-chainer-raw-0fa3d-65695c4586-f64qw.18a6deb9d69db1e9 namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: 44f6c622-43d9-468e-a44f-e3199ddc493b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-0fa3d} kind: Pod name: model-chainer-raw-0fa3d-65695c4586-f64qw namespace: kserve-ci-e2e-test resourceVersion: "15643" uid: 55501623-1e85-497c-8e95-44fd6fbfbe0c kind: Event lastTimestamp: "2026-04-16T15:06:32Z" message: Started container model-chainer-raw-0fa3d metadata: creationTimestamp: "2026-04-16T15:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:32Z" name: model-chainer-raw-0fa3d-65695c4586-f64qw.18a6deb9d715eb91 namespace: kserve-ci-e2e-test resourceVersion: "15696" uid: bf1d3ee9-f3ca-4b06-90b5-3c1194bc5c51 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-0fa3d} kind: Pod name: model-chainer-raw-0fa3d-65695c4586-f64qw namespace: kserve-ci-e2e-test resourceVersion: "15643" uid: 55501623-1e85-497c-8e95-44fd6fbfbe0c kind: Event lastTimestamp: "2026-04-16T15:06:38Z" message: Stopping container model-chainer-raw-0fa3d metadata: creationTimestamp: "2026-04-16T15:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:38Z" name: model-chainer-raw-0fa3d-65695c4586-f64qw.18a6debb409ea823 namespace: kserve-ci-e2e-test resourceVersion: "15763" uid: da7c7e8a-685e-4d54-ac7e-0fcc8c5d95c9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T15:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-0fa3d} kind: Pod name: model-chainer-raw-0fa3d-65695c4586-f64qw namespace: kserve-ci-e2e-test resourceVersion: "15643" uid: 55501623-1e85-497c-8e95-44fd6fbfbe0c kind: Event lastTimestamp: "2026-04-16T15:07:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T15:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:08Z" name: model-chainer-raw-0fa3d-65695c4586-f64qw.18a6debb40f2281f namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 9cc25be0-3757-480a-9707-1ddb48d0e91c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-0fa3d-65695c4586 namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: 6ce85146-be31-4480-8c61-f22294ff6a8b kind: Event lastTimestamp: "2026-04-16T15:06:28Z" message: 'Created pod: model-chainer-raw-0fa3d-65695c4586-f64qw' metadata: creationTimestamp: "2026-04-16T15:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:28Z" name: model-chainer-raw-0fa3d-65695c4586.18a6deb8e790797f namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: b6b6eea9-747e-4650-ae35-8501404bc62a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-0fa3d namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: 2942e3d8-64dc-48ed-b9e5-8bc9e053962e kind: Event lastTimestamp: "2026-04-16T15:06:28Z" message: Scaled up replica set model-chainer-raw-0fa3d-65695c4586 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:28Z" name: model-chainer-raw-0fa3d.18a6deb8e6e87b88 namespace: kserve-ci-e2e-test resourceVersion: "15639" uid: 8166077f-0726-4c31-8dc7-993b860de455 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-0fa3d namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: 41bd7d4d-d36c-48d2-83d6-d71af3478343 kind: Event lastTimestamp: "2026-04-16T15:06:28Z" message: InferenceGraph [model-chainer-raw-0fa3d] is Ready metadata: creationTimestamp: "2026-04-16T15:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:06:28Z" name: model-chainer-raw-0fa3d.18a6deb8eb8301ac namespace: kserve-ci-e2e-test resourceVersion: "15656" uid: 326bf6ab-62d4-4707-878c-6817a5785713 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:08:09.091480Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-007f8-774dcbc4f8-4hnw6 namespace: kserve-ci-e2e-test resourceVersion: "16602" uid: 8f0e4d64-6552-422e-b759-ef7870748314 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-007f8-774dcbc4f8-4hnw6 to ip-10-0-142-46.ec2.internal metadata: creationTimestamp: "2026-04-16T15:08:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:08:09Z" name: model-chainer-raw-hpa-007f8-774dcbc4f8-4hnw6.18a6ded042522c49 namespace: kserve-ci-e2e-test resourceVersion: "16608" uid: 15d5c51a-e4cf-429d-bb3e-cd9c251ea304 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-794cffb559-pkg6f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:09Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-007f8-774dcbc4f8-4hnw6 namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 8f0e4d64-6552-422e-b759-ef7870748314 kind: Event lastTimestamp: "2026-04-16T15:08:09Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:08:09Z" name: model-chainer-raw-hpa-007f8-774dcbc4f8-4hnw6.18a6ded05b4fdc64 namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: cab7c00e-74fa-4fbf-a836-8091d22637e6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-007f8} kind: Pod name: model-chainer-raw-hpa-007f8-774dcbc4f8-4hnw6 namespace: kserve-ci-e2e-test resourceVersion: "16606" uid: 8f0e4d64-6552-422e-b759-ef7870748314 kind: Event lastTimestamp: "2026-04-16T15:08:09Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:daaaf0b7769b899c00520a668e190161eefaa93b6eb75bfb8161ae18d89812c6" already present on machine metadata: creationTimestamp: "2026-04-16T15:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:09Z" name: model-chainer-raw-hpa-007f8-774dcbc4f8-4hnw6.18a6ded05c9c17ae namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: b4ace8e5-a86b-4710-9b25-2dc42c273f71 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-007f8} kind: Pod name: model-chainer-raw-hpa-007f8-774dcbc4f8-4hnw6 namespace: kserve-ci-e2e-test resourceVersion: "16606" uid: 8f0e4d64-6552-422e-b759-ef7870748314 kind: Event lastTimestamp: "2026-04-16T15:08:09Z" message: 'Created container: model-chainer-raw-hpa-007f8' metadata: creationTimestamp: "2026-04-16T15:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:09Z" name: model-chainer-raw-hpa-007f8-774dcbc4f8-4hnw6.18a6ded05f342431 namespace: kserve-ci-e2e-test resourceVersion: "16629" uid: f7a5a051-a5b0-407d-95e6-44d2e4a5dec6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-007f8} kind: Pod name: model-chainer-raw-hpa-007f8-774dcbc4f8-4hnw6 namespace: kserve-ci-e2e-test resourceVersion: "16606" uid: 8f0e4d64-6552-422e-b759-ef7870748314 kind: Event lastTimestamp: "2026-04-16T15:08:09Z" message: Started container model-chainer-raw-hpa-007f8 metadata: creationTimestamp: "2026-04-16T15:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:09Z" name: model-chainer-raw-hpa-007f8-774dcbc4f8-4hnw6.18a6ded05fb91c7c namespace: kserve-ci-e2e-test resourceVersion: "16630" uid: 6d9e38e0-71cc-489b-8d6e-ad82a0f554dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-007f8} kind: Pod name: model-chainer-raw-hpa-007f8-774dcbc4f8-4hnw6 namespace: kserve-ci-e2e-test resourceVersion: "16606" uid: 8f0e4d64-6552-422e-b759-ef7870748314 kind: Event lastTimestamp: "2026-04-16T15:08:19Z" message: Stopping container model-chainer-raw-hpa-007f8 metadata: creationTimestamp: "2026-04-16T15:08:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:19Z" name: model-chainer-raw-hpa-007f8-774dcbc4f8-4hnw6.18a6ded29a8c2947 namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: c94e0991-f3cb-48f9-ac92-8bca53fff9f4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:08:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-007f8} kind: Pod name: model-chainer-raw-hpa-007f8-774dcbc4f8-4hnw6 namespace: kserve-ci-e2e-test resourceVersion: "16606" uid: 8f0e4d64-6552-422e-b759-ef7870748314 kind: Event lastTimestamp: "2026-04-16T15:08:46Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T15:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:46Z" name: model-chainer-raw-hpa-007f8-774dcbc4f8-4hnw6.18a6ded312a2d9fa namespace: kserve-ci-e2e-test resourceVersion: "17080" uid: 663b7813-3658-4c92-8630-82d2ae9ec4b6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-007f8-774dcbc4f8 namespace: kserve-ci-e2e-test resourceVersion: "16597" uid: e7f67352-10e5-4f15-aca3-d5864d6c0460 kind: Event lastTimestamp: "2026-04-16T15:08:09Z" message: 'Created pod: model-chainer-raw-hpa-007f8-774dcbc4f8-4hnw6' metadata: creationTimestamp: "2026-04-16T15:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:09Z" name: model-chainer-raw-hpa-007f8-774dcbc4f8.18a6ded041f38ef7 namespace: kserve-ci-e2e-test resourceVersion: "16605" uid: 23a50832-3158-4fd7-ba2e-73bf1f65a4a0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-007f8 namespace: kserve-ci-e2e-test resourceVersion: "16595" uid: 23d638ae-563f-41f0-a65c-2a0870c7d528 kind: Event lastTimestamp: "2026-04-16T15:08:09Z" message: Scaled up replica set model-chainer-raw-hpa-007f8-774dcbc4f8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:09Z" name: model-chainer-raw-hpa-007f8.18a6ded040f2d2c0 namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: 74721d94-f8dc-4428-8a9b-545510113881 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-007f8 namespace: kserve-ci-e2e-test resourceVersion: "16616" uid: 202afda8-9058-4790-8e28-8745b7bdb457 kind: Event lastTimestamp: "2026-04-16T15:08:09Z" message: InferenceGraph [model-chainer-raw-hpa-007f8] is Ready metadata: creationTimestamp: "2026-04-16T15:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:08:09Z" name: model-chainer-raw-hpa-007f8.18a6ded044b373aa namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: 3c6bf8df-6fa2-413a-a7a8-e9b2d7dfef66 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:14:11.361390Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p namespace: kserve-ci-e2e-test resourceVersion: "19795" uid: 053841b9-fccf-41c9-a494-f9030a7432d7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p to ip-10-0-142-46.ec2.internal metadata: creationTimestamp: "2026-04-16T15:14:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:14:11Z" name: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p.18a6df249b4a6156 namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: 3e97879a-9313-4725-b963-728e53bc80bb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-794cffb559-pkg6f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:14:11Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p namespace: kserve-ci-e2e-test resourceVersion: "19808" uid: 053841b9-fccf-41c9-a494-f9030a7432d7 kind: Event lastTimestamp: "2026-04-16T15:14:11Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:14:11Z" name: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p.18a6df24b47d0f82 namespace: kserve-ci-e2e-test resourceVersion: "19821" uid: ee3111b8-eb1e-4be4-8347-b0f9a83c9557 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p namespace: kserve-ci-e2e-test resourceVersion: "19798" uid: 053841b9-fccf-41c9-a494-f9030a7432d7 kind: Event lastTimestamp: "2026-04-16T15:14:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:c49c84e69ddbb36db782821d4aa3d203f458a00c0c04dd506eb662aae339bb8a" already present on machine metadata: creationTimestamp: "2026-04-16T15:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:14:11Z" name: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p.18a6df24b5e1e910 namespace: kserve-ci-e2e-test resourceVersion: "19824" uid: e4186303-af55-4e15-b9cb-fb0237ac5c0e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p namespace: kserve-ci-e2e-test resourceVersion: "19798" uid: 053841b9-fccf-41c9-a494-f9030a7432d7 kind: Event lastTimestamp: "2026-04-16T15:14:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T15:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:14:11Z" name: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p.18a6df24b93a21f5 namespace: kserve-ci-e2e-test resourceVersion: "19826" uid: 5247fd70-001e-4df6-afeb-45087f28bee2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p namespace: kserve-ci-e2e-test resourceVersion: "19798" uid: 053841b9-fccf-41c9-a494-f9030a7432d7 kind: Event lastTimestamp: "2026-04-16T15:14:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T15:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:14:11Z" name: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p.18a6df24b9afcab1 namespace: kserve-ci-e2e-test resourceVersion: "19827" uid: 08386b66-22c9-4b6e-bf59-fbba8f57788c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:14:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p namespace: kserve-ci-e2e-test resourceVersion: "19798" uid: 053841b9-fccf-41c9-a494-f9030a7432d7 kind: Event lastTimestamp: "2026-04-16T15:14:16Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:14:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:14:16Z" name: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p.18a6df25c1d1c28f namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: dcc59b49-0e53-429d-b938-802eaa3ba146 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:14:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p namespace: kserve-ci-e2e-test resourceVersion: "19798" uid: 053841b9-fccf-41c9-a494-f9030a7432d7 kind: Event lastTimestamp: "2026-04-16T15:14:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:14:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:14:16Z" name: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p.18a6df25c55910dc namespace: kserve-ci-e2e-test resourceVersion: "19869" uid: eda47318-fbc0-470f-8e29-764a63b270f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:14:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p namespace: kserve-ci-e2e-test resourceVersion: "19798" uid: 053841b9-fccf-41c9-a494-f9030a7432d7 kind: Event lastTimestamp: "2026-04-16T15:14:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:14:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:14:16Z" name: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p.18a6df25c5d2b1c6 namespace: kserve-ci-e2e-test resourceVersion: "19871" uid: 5d8459de-fe0f-41fe-9527-8659ed9a02e4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T15:14:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p namespace: kserve-ci-e2e-test resourceVersion: "19798" uid: 053841b9-fccf-41c9-a494-f9030a7432d7 kind: Event lastTimestamp: "2026-04-16T15:15:33Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:14:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:15:33Z" name: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p.18a6df25fdb7cfb7 namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: f96b4f94-9b2a-4179-bd88-3a85d0e1aabd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:15:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p namespace: kserve-ci-e2e-test resourceVersion: "19798" uid: 053841b9-fccf-41c9-a494-f9030a7432d7 kind: Event lastTimestamp: "2026-04-16T15:15:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:15:31Z" name: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p.18a6df3743b18964 namespace: kserve-ci-e2e-test resourceVersion: "20439" uid: ca3887c2-2606-48b4-b15a-e341ec5d6101 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:14:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-7eb70-predictor-8458bdd6cb namespace: kserve-ci-e2e-test resourceVersion: "19787" uid: bd8fcde0-83d4-4e09-9644-8c4072d6e5b4 kind: Event lastTimestamp: "2026-04-16T15:14:11Z" message: 'Created pod: raw-sklearn-7eb70-predictor-8458bdd6cb-ntq7p' metadata: creationTimestamp: "2026-04-16T15:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:14:11Z" name: raw-sklearn-7eb70-predictor-8458bdd6cb.18a6df249ab0dfa0 namespace: kserve-ci-e2e-test resourceVersion: "19797" uid: 8c9ae29c-def2-40c2-b98b-65854e8a73c7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:14:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-7eb70-predictor namespace: kserve-ci-e2e-test resourceVersion: "19784" uid: 09379494-df58-4ebe-8055-2eccd5cfe6bf kind: Event lastTimestamp: "2026-04-16T15:14:11Z" message: Scaled up replica set raw-sklearn-7eb70-predictor-8458bdd6cb from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:14:11Z" name: raw-sklearn-7eb70-predictor.18a6df2498260c89 namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 42c43fa2-cc28-4c58-b0bf-5d6f3a3726ba reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:14:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-7eb70-predictor namespace: kserve-ci-e2e-test resourceVersion: "19786" uid: 987e0b08-1bff-4cfa-9551-f5ad718d4677 kind: Event lastTimestamp: "2026-04-16T15:14:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:14:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:14:26Z" name: raw-sklearn-7eb70-predictor.18a6df281694c615 namespace: kserve-ci-e2e-test resourceVersion: "19946" uid: 02df56ab-e530-43ef-9bdb-81a98d5fc43f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:14:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-7eb70-predictor namespace: kserve-ci-e2e-test resourceVersion: "19786" uid: 987e0b08-1bff-4cfa-9551-f5ad718d4677 kind: Event lastTimestamp: "2026-04-16T15:14:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:14:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:14:26Z" name: raw-sklearn-7eb70-predictor.18a6df2816ed75d2 namespace: kserve-ci-e2e-test resourceVersion: "19947" uid: 144cdb38-6297-4ae6-b1cb-07bba2e2843e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T15:14:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-7eb70-predictor namespace: kserve-ci-e2e-test resourceVersion: "19945" uid: 987e0b08-1bff-4cfa-9551-f5ad718d4677 kind: Event lastTimestamp: "2026-04-16T15:15:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:14:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:15:26Z" name: raw-sklearn-7eb70-predictor.18a6df2b96172fc1 namespace: kserve-ci-e2e-test resourceVersion: "20375" uid: 211c5de6-874d-4d1d-9816-28eabfb4221f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T15:14:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-7eb70-predictor namespace: kserve-ci-e2e-test resourceVersion: "19945" uid: 987e0b08-1bff-4cfa-9551-f5ad718d4677 kind: Event lastTimestamp: "2026-04-16T15:15:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:14:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:15:26Z" name: raw-sklearn-7eb70-predictor.18a6df2b967a3b51 namespace: kserve-ci-e2e-test resourceVersion: "20376" uid: 08d3840a-69d3-4610-8e55-1a77111f1b4e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T15:14:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-7eb70 namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: f7908bbe-7cbd-4fba-bf31-a38989dc90ac kind: Event lastTimestamp: "2026-04-16T15:15:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:15:23Z" name: raw-sklearn-7eb70.18a6df2497ec4a06 namespace: kserve-ci-e2e-test resourceVersion: "20354" uid: 3c16947c-9986-44c9-a8a9-d4e61dbe9cdd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:14:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-7eb70 namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: f7908bbe-7cbd-4fba-bf31-a38989dc90ac kind: Event lastTimestamp: "2026-04-16T15:14:11Z" message: 'Failed to update status for InferenceService "raw-sklearn-7eb70": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-7eb70": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:14:11Z" name: raw-sklearn-7eb70.18a6df24987abbea namespace: kserve-ci-e2e-test resourceVersion: "19811" uid: e846380c-3631-4f24-bd64-e9b80e2a2edd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:14:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-7eb70 namespace: kserve-ci-e2e-test resourceVersion: "19793" uid: f7908bbe-7cbd-4fba-bf31-a38989dc90ac kind: Event lastTimestamp: "2026-04-16T15:14:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-7eb70": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:14:11Z" name: raw-sklearn-7eb70.18a6df249e66d536 namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: afc82c9b-cf66-4c7e-8e80-7b27f3f0a998 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:15:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-7eb70 namespace: kserve-ci-e2e-test resourceVersion: "20352" uid: f7908bbe-7cbd-4fba-bf31-a38989dc90ac kind: Event lastTimestamp: "2026-04-16T15:15:23Z" message: InferenceService [raw-sklearn-7eb70] is Ready metadata: creationTimestamp: "2026-04-16T15:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:15:23Z" name: raw-sklearn-7eb70.18a6df356004978e namespace: kserve-ci-e2e-test resourceVersion: "20353" uid: 54ec2024-8a13-461a-a98c-967e271f09e4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:15:31.550946Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt namespace: kserve-ci-e2e-test resourceVersion: "20449" uid: 41446b76-2e24-47db-a963-c8287e69fbbd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt to ip-10-0-142-46.ec2.internal metadata: creationTimestamp: "2026-04-16T15:15:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:15:31Z" name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt.18a6df3746f6a553 namespace: kserve-ci-e2e-test resourceVersion: "20453" uid: be0a8a53-3595-49a6-9fc2-2dfd56e917c1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-794cffb559-pkg6f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:15:31Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt namespace: kserve-ci-e2e-test resourceVersion: "20457" uid: 41446b76-2e24-47db-a963-c8287e69fbbd kind: Event lastTimestamp: "2026-04-16T15:15:31Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:15:31Z" name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt.18a6df375ff483ac namespace: kserve-ci-e2e-test resourceVersion: "20471" uid: 278de1b1-6a66-485b-859e-ca958768a6d3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:15:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: 41446b76-2e24-47db-a963-c8287e69fbbd kind: Event lastTimestamp: "2026-04-16T15:15:31Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:c49c84e69ddbb36db782821d4aa3d203f458a00c0c04dd506eb662aae339bb8a" already present on machine metadata: creationTimestamp: "2026-04-16T15:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:15:31Z" name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt.18a6df376132491d namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: 0dabe672-78fd-4c68-8aa8-b9bf33f48d81 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:15:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: 41446b76-2e24-47db-a963-c8287e69fbbd kind: Event lastTimestamp: "2026-04-16T15:15:32Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T15:15:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:15:32Z" name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt.18a6df376439aa7d namespace: kserve-ci-e2e-test resourceVersion: "20475" uid: ff86a733-7533-4bfe-bb34-2b238736e96f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:15:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: 41446b76-2e24-47db-a963-c8287e69fbbd kind: Event lastTimestamp: "2026-04-16T15:15:32Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T15:15:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:15:32Z" name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt.18a6df3764ac50c7 namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: 8319f5f9-7de0-4b3e-aefb-26da8befe9da reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:15:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: 41446b76-2e24-47db-a963-c8287e69fbbd kind: Event lastTimestamp: "2026-04-16T15:15:36Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:15:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:15:36Z" name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt.18a6df386fcda04f namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: 25b973aa-e6d0-48ed-a000-0666597851fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:15:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: 41446b76-2e24-47db-a963-c8287e69fbbd kind: Event lastTimestamp: "2026-04-16T15:15:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:15:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:15:36Z" name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt.18a6df3872d69514 namespace: kserve-ci-e2e-test resourceVersion: "20522" uid: c06ad3cf-d9e1-44c7-be4e-fbae3be12e6a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:15:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: 41446b76-2e24-47db-a963-c8287e69fbbd kind: Event lastTimestamp: "2026-04-16T15:15:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:15:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:15:36Z" name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt.18a6df38734a399a namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: 3f59190a-0fff-4afb-b7ed-7809912f3440 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T15:15:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: 41446b76-2e24-47db-a963-c8287e69fbbd kind: Event lastTimestamp: "2026-04-16T15:16:39Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:15:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:39Z" name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt.18a6df38abc5b0b4 namespace: kserve-ci-e2e-test resourceVersion: "20971" uid: f1b7aa89-f1f0-4d78-b964-8da8305947cd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: 41446b76-2e24-47db-a963-c8287e69fbbd kind: Event lastTimestamp: "2026-04-16T15:16:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:16:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:51Z" name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt.18a6df49ee33f4a0 namespace: kserve-ci-e2e-test resourceVersion: "21078" uid: 3d4e4858-a3d2-47b2-bb5b-282cefe6212f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-46.ec2.internal source: component: kubelet host: ip-10-0-142-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:15:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 85b7c0fa-783d-4f47-bbb5-ced7c5b0d304 kind: Event lastTimestamp: "2026-04-16T15:15:31Z" message: 'Created pod: raw-sklearn-runtime-d8ecb-predictor-5ffd647596-dr7xt' metadata: creationTimestamp: "2026-04-16T15:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:15:31Z" name: raw-sklearn-runtime-d8ecb-predictor-5ffd647596.18a6df37467f86ef namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: 9d083551-5a8c-4b8a-87db-ccac97da26cb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:15:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-d8ecb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: ac649bba-66f9-4f32-8752-a486a8ac635c kind: Event lastTimestamp: "2026-04-16T15:15:31Z" message: Scaled up replica set raw-sklearn-runtime-d8ecb-predictor-5ffd647596 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:15:31Z" name: raw-sklearn-runtime-d8ecb-predictor.18a6df3743e974b1 namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 8afaefd7-4772-42e4-8db3-f36da820f5a0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:15:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-d8ecb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20437" uid: 2db9c7c8-8295-4ffe-80a4-482bb23f56af kind: Event lastTimestamp: "2026-04-16T15:16:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:16:16Z" name: raw-sklearn-runtime-d8ecb-predictor.18a6df3ac2aeed92 namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: b31b38f2-ce53-4227-89f9-4db0a031aa90 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:15:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-d8ecb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20437" uid: 2db9c7c8-8295-4ffe-80a4-482bb23f56af kind: Event lastTimestamp: "2026-04-16T15:16:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:16:16Z" name: raw-sklearn-runtime-d8ecb-predictor.18a6df3ac2eb5234 namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: 68686433-d78f-4170-bba8-487eddcdc225 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:16:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-d8ecb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 2db9c7c8-8295-4ffe-80a4-482bb23f56af kind: Event lastTimestamp: "2026-04-16T15:16:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:16:46Z" name: raw-sklearn-runtime-d8ecb-predictor.18a6df453f9807ea namespace: kserve-ci-e2e-test resourceVersion: "21019" uid: 8f45efad-dfa3-4330-99be-a01f56a5f7c9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:16:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-d8ecb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 2db9c7c8-8295-4ffe-80a4-482bb23f56af kind: Event lastTimestamp: "2026-04-16T15:16:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:16:46Z" name: raw-sklearn-runtime-d8ecb-predictor.18a6df453ff7930d namespace: kserve-ci-e2e-test resourceVersion: "21020" uid: e55d277f-aa4f-4d11-894f-0588b8506fff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-04-16T15:15:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-d8ecb namespace: kserve-ci-e2e-test resourceVersion: "20411" uid: ebfd0e17-c771-4064-9106-f5d4dbe5839e kind: Event lastTimestamp: "2026-04-16T15:16:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:16:49Z" name: raw-sklearn-runtime-d8ecb.18a6df3743c6851f namespace: kserve-ci-e2e-test resourceVersion: "21047" uid: 8c6061e3-dae8-483c-8a38-af1dacc26d14 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:15:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-d8ecb namespace: kserve-ci-e2e-test resourceVersion: "20411" uid: ebfd0e17-c771-4064-9106-f5d4dbe5839e kind: Event lastTimestamp: "2026-04-16T15:15:31Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-d8ecb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-d8ecb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:15:31Z" name: raw-sklearn-runtime-d8ecb.18a6df37441c2588 namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: a69fd4bd-6ff3-4db5-a802-4cf34bfc93f4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:15:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-d8ecb namespace: kserve-ci-e2e-test resourceVersion: "20420" uid: ebfd0e17-c771-4064-9106-f5d4dbe5839e kind: Event lastTimestamp: "2026-04-16T15:15:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-d8ecb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:15:31Z" name: raw-sklearn-runtime-d8ecb.18a6df3748b4dbcc namespace: kserve-ci-e2e-test resourceVersion: "20461" uid: 188ab54b-466e-467c-bee5-46e250f007df reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-d8ecb namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: ebfd0e17-c771-4064-9106-f5d4dbe5839e kind: Event lastTimestamp: "2026-04-16T15:16:49Z" message: InferenceService [raw-sklearn-runtime-d8ecb] is Ready metadata: creationTimestamp: "2026-04-16T15:16:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:16:49Z" name: raw-sklearn-runtime-d8ecb.18a6df49668bc6e3 namespace: kserve-ci-e2e-test resourceVersion: "21046" uid: 6c1968ee-bad6-4bfb-b8b2-67401f7bbc4a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "21266"