--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:53:00.973306Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-7ec1f4-predictor-6584dc5565-pvggw namespace: kserve-ci-e2e-test resourceVersion: "19854" uid: b75b79a9-b080-43fd-96ca-1452dc1b5bd1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-7ec1f4-predictor-6584dc5565-pvggw to ip-10-0-140-1.ec2.internal metadata: creationTimestamp: "2026-04-16T18:53:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:53:00Z" name: isvc-init-fail-7ec1f4-predictor-6584dc5565-pvggw.18a6eb1595250431 namespace: kserve-ci-e2e-test resourceVersion: "19859" uid: 3235fcf3-4405-4f6f-aa6f-4cd98a5a6560 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9d7c89bb-dzjdk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:53:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-7ec1f4-predictor-6584dc5565-pvggw namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: b75b79a9-b080-43fd-96ca-1452dc1b5bd1 kind: Event lastTimestamp: "2026-04-16T18:53:01Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:53:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:53:01Z" name: isvc-init-fail-7ec1f4-predictor-6584dc5565-pvggw.18a6eb15ae142897 namespace: kserve-ci-e2e-test resourceVersion: "19872" uid: 4a862cfd-88b5-4f6a-ab16-84faf807c67c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:53:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-7ec1f4-predictor-6584dc5565-pvggw namespace: kserve-ci-e2e-test resourceVersion: "19857" uid: b75b79a9-b080-43fd-96ca-1452dc1b5bd1 kind: Event lastTimestamp: "2026-04-16T18:53:04Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T18:53:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:53:04Z" name: isvc-init-fail-7ec1f4-predictor-6584dc5565-pvggw.18a6eb15af470771 namespace: kserve-ci-e2e-test resourceVersion: "19906" uid: 206e7b16-01b1-4c5c-ad7a-d62d084c5076 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:53:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-7ec1f4-predictor-6584dc5565-pvggw namespace: kserve-ci-e2e-test resourceVersion: "19857" uid: b75b79a9-b080-43fd-96ca-1452dc1b5bd1 kind: Event lastTimestamp: "2026-04-16T18:53:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:53:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:53:04Z" name: isvc-init-fail-7ec1f4-predictor-6584dc5565-pvggw.18a6eb15b27e9ea8 namespace: kserve-ci-e2e-test resourceVersion: "19910" uid: 54ccda06-4df7-4e13-8347-9aca3bd7921a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:53:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-7ec1f4-predictor-6584dc5565-pvggw namespace: kserve-ci-e2e-test resourceVersion: "19857" uid: b75b79a9-b080-43fd-96ca-1452dc1b5bd1 kind: Event lastTimestamp: "2026-04-16T18:53:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:53:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:53:04Z" name: isvc-init-fail-7ec1f4-predictor-6584dc5565-pvggw.18a6eb15b2ed0a9b namespace: kserve-ci-e2e-test resourceVersion: "19911" uid: bc838b5b-2cce-4108-b61b-f8a3c64df024 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:53:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-7ec1f4-predictor-6584dc5565-pvggw namespace: kserve-ci-e2e-test resourceVersion: "19857" uid: b75b79a9-b080-43fd-96ca-1452dc1b5bd1 kind: Event lastTimestamp: "2026-04-16T18:53:06Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-04-16T18:53:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:53:06Z" name: isvc-init-fail-7ec1f4-predictor-6584dc5565-pvggw.18a6eb16cda4dbfb namespace: kserve-ci-e2e-test resourceVersion: "19979" uid: 70e06ba5-a6cd-48fe-9f07-023f9a325ad8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:53:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-7ec1f4-predictor-6584dc5565 namespace: kserve-ci-e2e-test resourceVersion: "19839" uid: 532401d3-bea0-4f73-a4c8-90242b3b3710 kind: Event lastTimestamp: "2026-04-16T18:53:00Z" message: 'Created pod: isvc-init-fail-7ec1f4-predictor-6584dc5565-pvggw' metadata: creationTimestamp: "2026-04-16T18:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:53:00Z" name: isvc-init-fail-7ec1f4-predictor-6584dc5565.18a6eb15948e41b5 namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: f1366d6c-764b-4d36-b474-b8d118c9ae79 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:53:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-7ec1f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19834" uid: d5a9e27e-af11-49df-8ffc-d84a9f87862e kind: Event lastTimestamp: "2026-04-16T18:53:00Z" message: Scaled up replica set isvc-init-fail-7ec1f4-predictor-6584dc5565 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:53:00Z" name: isvc-init-fail-7ec1f4-predictor.18a6eb159194a219 namespace: kserve-ci-e2e-test resourceVersion: "19845" uid: 7593a02e-243e-4cc7-bf1b-1cff7d87b796 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T18:53:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-7ec1f4 namespace: kserve-ci-e2e-test resourceVersion: "19815" uid: f8782ce2-f386-45b6-aae6-56ba6c56068c kind: Event lastTimestamp: "2026-04-16T18:53:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:53:04Z" name: isvc-init-fail-7ec1f4.18a6eb15919a6f21 namespace: kserve-ci-e2e-test resourceVersion: "19912" uid: f378233b-c26b-43d6-8c09-b4d9ae8c856e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:53:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-7ec1f4 namespace: kserve-ci-e2e-test resourceVersion: "19815" uid: f8782ce2-f386-45b6-aae6-56ba6c56068c kind: Event lastTimestamp: "2026-04-16T18:53:00Z" message: 'Failed to update status for InferenceService "isvc-init-fail-7ec1f4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-7ec1f4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:53:00Z" name: isvc-init-fail-7ec1f4.18a6eb159245db80 namespace: kserve-ci-e2e-test resourceVersion: "19849" uid: e4b2c8de-d0a4-45b5-a863-05dd8fcbf0e1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:53:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-7ec1f4 namespace: kserve-ci-e2e-test resourceVersion: "19815" uid: f8782ce2-f386-45b6-aae6-56ba6c56068c kind: Event lastTimestamp: "2026-04-16T18:53:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-7ec1f4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:53:00Z" name: isvc-init-fail-7ec1f4.18a6eb15924631bc namespace: kserve-ci-e2e-test resourceVersion: "19850" uid: f543f61b-03f1-490d-aee0-a305cfd45533 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:47:29.121720Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp namespace: kserve-ci-e2e-test resourceVersion: "17078" uid: f03d9f60-6ac7-4bf8-9e7f-6d48ed4d1170 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-79732-predictor-f97f4f564-lwgbp to ip-10-0-140-1.ec2.internal metadata: creationTimestamp: "2026-04-16T18:47:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:47:29Z" name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp.18a6eac8513fc488 namespace: kserve-ci-e2e-test resourceVersion: "17083" uid: 3df43312-985f-4a9a-a866-2ef2b89f177c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9d7c89bb-dzjdk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:29Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp namespace: kserve-ci-e2e-test resourceVersion: "17088" uid: f03d9f60-6ac7-4bf8-9e7f-6d48ed4d1170 kind: Event lastTimestamp: "2026-04-16T18:47:29Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:47:29Z" name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp.18a6eac86a328a22 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: d164eab2-76f2-4576-97d8-488936416d7c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp namespace: kserve-ci-e2e-test resourceVersion: "17080" uid: f03d9f60-6ac7-4bf8-9e7f-6d48ed4d1170 kind: Event lastTimestamp: "2026-04-16T18:47:29Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T18:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:29Z" name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp.18a6eac86bf3d442 namespace: kserve-ci-e2e-test resourceVersion: "17096" uid: 200d9def-7814-4735-8a65-623b0f1bf694 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp namespace: kserve-ci-e2e-test resourceVersion: "17080" uid: f03d9f60-6ac7-4bf8-9e7f-6d48ed4d1170 kind: Event lastTimestamp: "2026-04-16T18:47:29Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:29Z" name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp.18a6eac86f5aa858 namespace: kserve-ci-e2e-test resourceVersion: "17099" uid: 99b1e881-3739-4d22-bcc4-f10b64fcb0b5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp namespace: kserve-ci-e2e-test resourceVersion: "17080" uid: f03d9f60-6ac7-4bf8-9e7f-6d48ed4d1170 kind: Event lastTimestamp: "2026-04-16T18:47:29Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:29Z" name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp.18a6eac86fcd1be7 namespace: kserve-ci-e2e-test resourceVersion: "17100" uid: 8e3b8fd7-3dce-49b6-a062-a618b6b8180f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp namespace: kserve-ci-e2e-test resourceVersion: "17080" uid: f03d9f60-6ac7-4bf8-9e7f-6d48ed4d1170 kind: Event lastTimestamp: "2026-04-16T18:47:34Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1326" already present on machine metadata: creationTimestamp: "2026-04-16T18:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:34Z" name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp.18a6eac977192ece namespace: kserve-ci-e2e-test resourceVersion: "17141" uid: 60ed9839-acf7-4be2-b3fc-db7fde745f60 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp namespace: kserve-ci-e2e-test resourceVersion: "17080" uid: f03d9f60-6ac7-4bf8-9e7f-6d48ed4d1170 kind: Event lastTimestamp: "2026-04-16T18:47:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:34Z" name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp.18a6eac97adb3bfe namespace: kserve-ci-e2e-test resourceVersion: "17145" uid: 3f8466fb-4bc3-4755-81b7-3d690c5b44b0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp namespace: kserve-ci-e2e-test resourceVersion: "17080" uid: f03d9f60-6ac7-4bf8-9e7f-6d48ed4d1170 kind: Event lastTimestamp: "2026-04-16T18:47:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:34Z" name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp.18a6eac97b53482e namespace: kserve-ci-e2e-test resourceVersion: "17146" uid: 9da4a966-8072-4681-b46c-ada63050d274 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp namespace: kserve-ci-e2e-test resourceVersion: "17080" uid: f03d9f60-6ac7-4bf8-9e7f-6d48ed4d1170 kind: Event lastTimestamp: "2026-04-16T18:47:34Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:f2fb11757d99ef46b1388e1609ace6b5f516fdebd4ee81c172186c5de9dc18b9" already present on machine metadata: creationTimestamp: "2026-04-16T18:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:34Z" name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp.18a6eac97b5fd8b3 namespace: kserve-ci-e2e-test resourceVersion: "17147" uid: 872d66f2-79c7-4a88-b1b3-17b95d9fa60f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp namespace: kserve-ci-e2e-test resourceVersion: "17080" uid: f03d9f60-6ac7-4bf8-9e7f-6d48ed4d1170 kind: Event lastTimestamp: "2026-04-16T18:47:34Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-16T18:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:34Z" name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp.18a6eac97e1b22a8 namespace: kserve-ci-e2e-test resourceVersion: "17149" uid: 13f78513-ec5b-4a20-bfcf-b4867afcc496 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp namespace: kserve-ci-e2e-test resourceVersion: "17080" uid: f03d9f60-6ac7-4bf8-9e7f-6d48ed4d1170 kind: Event lastTimestamp: "2026-04-16T18:47:34Z" message: Started container agent metadata: creationTimestamp: "2026-04-16T18:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:34Z" name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp.18a6eac97e8b3023 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: fb41868b-4196-499c-bfc1-495948d37fe0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-16T18:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp namespace: kserve-ci-e2e-test resourceVersion: "17080" uid: f03d9f60-6ac7-4bf8-9e7f-6d48ed4d1170 kind: Event lastTimestamp: "2026-04-16T18:49:16Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:16Z" name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp.18a6eac9b3113fec namespace: kserve-ci-e2e-test resourceVersion: "17987" uid: 3847e3e8-d64b-4d4b-a460-ee868207c0bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-16T18:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp namespace: kserve-ci-e2e-test resourceVersion: "17080" uid: f03d9f60-6ac7-4bf8-9e7f-6d48ed4d1170 kind: Event lastTimestamp: "2026-04-16T18:49:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:16Z" name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp.18a6eac9b31b05de namespace: kserve-ci-e2e-test resourceVersion: "17988" uid: 152feaff-d7e9-401e-a13d-dd310db54427 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp namespace: kserve-ci-e2e-test resourceVersion: "17080" uid: f03d9f60-6ac7-4bf8-9e7f-6d48ed4d1170 kind: Event lastTimestamp: "2026-04-16T18:48:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:48:54Z" name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp.18a6eadc2bfefe1c namespace: kserve-ci-e2e-test resourceVersion: "17772" uid: f72a07f8-32c9-4704-9036-fc8a1622b494 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp namespace: kserve-ci-e2e-test resourceVersion: "17080" uid: f03d9f60-6ac7-4bf8-9e7f-6d48ed4d1170 kind: Event lastTimestamp: "2026-04-16T18:48:54Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-16T18:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:48:54Z" name: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp.18a6eadc2c000ad5 namespace: kserve-ci-e2e-test resourceVersion: "17773" uid: 1381cdb2-ada1-46bc-bc82-f6c3ac292260 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-79732-predictor-f97f4f564 namespace: kserve-ci-e2e-test resourceVersion: "17067" uid: d69e1f62-6eb4-43d5-a861-50b2fb08da4e kind: Event lastTimestamp: "2026-04-16T18:47:29Z" message: 'Created pod: isvc-logger-raw-79732-predictor-f97f4f564-lwgbp' metadata: creationTimestamp: "2026-04-16T18:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:47:29Z" name: isvc-logger-raw-79732-predictor-f97f4f564.18a6eac850ecbba9 namespace: kserve-ci-e2e-test resourceVersion: "17081" uid: b1e8b0d9-d29e-4827-a62e-2ded74c91a5a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-79732-predictor namespace: kserve-ci-e2e-test resourceVersion: "17065" uid: 982ec525-02f4-4341-bc74-ab3713dd6d4a kind: Event lastTimestamp: "2026-04-16T18:47:29Z" message: Scaled up replica set isvc-logger-raw-79732-predictor-f97f4f564 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:47:29Z" name: isvc-logger-raw-79732-predictor.18a6eac84d8edab2 namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: da8bae9f-d052-4529-bf37-1134ade19e8a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:47:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-79732-predictor namespace: kserve-ci-e2e-test resourceVersion: "17066" uid: 61e173c1-6a52-4269-b61f-e3d17ab4bc8e kind: Event lastTimestamp: "2026-04-16T18:47:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:47:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:47:59Z" name: isvc-logger-raw-79732-predictor.18a6eacbcc095505 namespace: kserve-ci-e2e-test resourceVersion: "17331" uid: bb931b0a-0a9f-40c7-9490-3497713f3853 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:47:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-79732-predictor namespace: kserve-ci-e2e-test resourceVersion: "17066" uid: 61e173c1-6a52-4269-b61f-e3d17ab4bc8e kind: Event lastTimestamp: "2026-04-16T18:47:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:47:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:47:59Z" name: isvc-logger-raw-79732-predictor.18a6eacbcc741356 namespace: kserve-ci-e2e-test resourceVersion: "17332" uid: fa6d1e45-db35-4eb6-ba8d-0be279a85b49 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:48:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-79732-predictor namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: 61e173c1-6a52-4269-b61f-e3d17ab4bc8e kind: Event lastTimestamp: "2026-04-16T18:48:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:48:44Z" name: isvc-logger-raw-79732-predictor.18a6ead2ca5272fa namespace: kserve-ci-e2e-test resourceVersion: "17658" uid: e42f9308-8aae-4b01-9c90-910ea5a0cbb5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:48:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-79732-predictor namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: 61e173c1-6a52-4269-b61f-e3d17ab4bc8e kind: Event lastTimestamp: "2026-04-16T18:48:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:48:44Z" name: isvc-logger-raw-79732-predictor.18a6ead2cacb5be1 namespace: kserve-ci-e2e-test resourceVersion: "17659" uid: c83d83b3-107b-4094-b2ae-68dc096f4c9f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T18:47:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-79732 namespace: kserve-ci-e2e-test resourceVersion: "17056" uid: f79b28c8-780f-4290-8840-1dce89a1ac2e kind: Event lastTimestamp: "2026-04-16T18:48:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:48:46Z" name: isvc-logger-raw-79732.18a6eac84d30c341 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: aad04d2f-8ca3-42c3-9471-a01532266f65 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-79732 namespace: kserve-ci-e2e-test resourceVersion: "17056" uid: f79b28c8-780f-4290-8840-1dce89a1ac2e kind: Event lastTimestamp: "2026-04-16T18:47:29Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-79732": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-79732": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:47:29Z" name: isvc-logger-raw-79732.18a6eac84d9db1e2 namespace: kserve-ci-e2e-test resourceVersion: "17071" uid: c5466139-3daa-4552-8012-cd10ac7e2f98 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-79732 namespace: kserve-ci-e2e-test resourceVersion: "17678" uid: f79b28c8-780f-4290-8840-1dce89a1ac2e kind: Event lastTimestamp: "2026-04-16T18:48:46Z" message: InferenceService [isvc-logger-raw-79732] is Ready metadata: creationTimestamp: "2026-04-16T18:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:48:46Z" name: isvc-logger-raw-79732.18a6eada3fe47409 namespace: kserve-ci-e2e-test resourceVersion: "17679" uid: ea8613cc-ec8f-4e13-880c-ce976d57daa5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:51:24.656178Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8 namespace: kserve-ci-e2e-test resourceVersion: "19007" uid: da0daa06-f6df-4f51-95c7-98d163d34200 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8 to ip-10-0-140-1.ec2.internal metadata: creationTimestamp: "2026-04-16T18:51:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:51:24Z" name: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8.18a6eaff28325334 namespace: kserve-ci-e2e-test resourceVersion: "19013" uid: 22be8053-51b7-42f2-9296-79ba1776d668 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9d7c89bb-dzjdk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:51:25Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8 namespace: kserve-ci-e2e-test resourceVersion: "19018" uid: da0daa06-f6df-4f51-95c7-98d163d34200 kind: Event lastTimestamp: "2026-04-16T18:51:25Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:51:25Z" name: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8.18a6eaff4d78f79c namespace: kserve-ci-e2e-test resourceVersion: "19027" uid: c0490826-51b2-45cd-a565-1deffb2d7e98 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:51:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8 namespace: kserve-ci-e2e-test resourceVersion: "19009" uid: da0daa06-f6df-4f51-95c7-98d163d34200 kind: Event lastTimestamp: "2026-04-16T18:51:25Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T18:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:51:25Z" name: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8.18a6eaff4ebd5f25 namespace: kserve-ci-e2e-test resourceVersion: "19029" uid: b624a6a8-2eb3-416c-8b60-8b3f2f4a73cd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:51:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8 namespace: kserve-ci-e2e-test resourceVersion: "19009" uid: da0daa06-f6df-4f51-95c7-98d163d34200 kind: Event lastTimestamp: "2026-04-16T18:51:25Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:51:25Z" name: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8.18a6eaff51fa424c namespace: kserve-ci-e2e-test resourceVersion: "19031" uid: 6866089c-771b-48d9-a4fa-f33b02325ce4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:51:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8 namespace: kserve-ci-e2e-test resourceVersion: "19009" uid: da0daa06-f6df-4f51-95c7-98d163d34200 kind: Event lastTimestamp: "2026-04-16T18:51:25Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:51:25Z" name: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8.18a6eaff5266ca51 namespace: kserve-ci-e2e-test resourceVersion: "19032" uid: 1e65f2c4-e871-47d5-a678-59bbf8b68a04 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:51:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8 namespace: kserve-ci-e2e-test resourceVersion: "19009" uid: da0daa06-f6df-4f51-95c7-98d163d34200 kind: Event lastTimestamp: "2026-04-16T18:51:29Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1326" already present on machine metadata: creationTimestamp: "2026-04-16T18:51:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:51:29Z" name: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8.18a6eb005f2403b9 namespace: kserve-ci-e2e-test resourceVersion: "19068" uid: 946e1e8d-4b24-40bc-8c6c-7af2b069716d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:51:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8 namespace: kserve-ci-e2e-test resourceVersion: "19009" uid: da0daa06-f6df-4f51-95c7-98d163d34200 kind: Event lastTimestamp: "2026-04-16T18:51:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:51:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:51:29Z" name: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8.18a6eb0062396768 namespace: kserve-ci-e2e-test resourceVersion: "19072" uid: 92bf62b9-3a5c-440e-bdc8-17b2c2fb02ee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:51:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8 namespace: kserve-ci-e2e-test resourceVersion: "19009" uid: da0daa06-f6df-4f51-95c7-98d163d34200 kind: Event lastTimestamp: "2026-04-16T18:51:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:51:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:51:29Z" name: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8.18a6eb0062abcd52 namespace: kserve-ci-e2e-test resourceVersion: "19074" uid: caa4cf19-48d0-46f7-b69e-00bac1f7abe6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T18:51:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8 namespace: kserve-ci-e2e-test resourceVersion: "19009" uid: da0daa06-f6df-4f51-95c7-98d163d34200 kind: Event lastTimestamp: "2026-04-16T18:53:01Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:51:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:53:01Z" name: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8.18a6eb009b12f3db namespace: kserve-ci-e2e-test resourceVersion: "19883" uid: c98853c4-dd68-4d5f-9b8f-1e6e1bcae9de reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:53:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8 namespace: kserve-ci-e2e-test resourceVersion: "19009" uid: da0daa06-f6df-4f51-95c7-98d163d34200 kind: Event lastTimestamp: "2026-04-16T18:53:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:53:00Z" name: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8.18a6eb1592aed973 namespace: kserve-ci-e2e-test resourceVersion: "19852" uid: be0a5bf0-edd1-4a5f-844c-323db2ce3206 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:51:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-2e895d-predictor-6b8ff99d46 namespace: kserve-ci-e2e-test resourceVersion: "18997" uid: 08be1681-d0aa-44da-9a43-4fb51a5dad28 kind: Event lastTimestamp: "2026-04-16T18:51:24Z" message: 'Created pod: isvc-primary-2e895d-predictor-6b8ff99d46-ghzg8' metadata: creationTimestamp: "2026-04-16T18:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:51:24Z" name: isvc-primary-2e895d-predictor-6b8ff99d46.18a6eaff27ba133a namespace: kserve-ci-e2e-test resourceVersion: "19008" uid: efb7fdb5-e3b6-461e-93f1-a50478603e79 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:51:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-2e895d-predictor namespace: kserve-ci-e2e-test resourceVersion: "18992" uid: 1f6fa517-006c-40bc-953b-01db062fdcaa kind: Event lastTimestamp: "2026-04-16T18:51:24Z" message: Scaled up replica set isvc-primary-2e895d-predictor-6b8ff99d46 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:51:24Z" name: isvc-primary-2e895d-predictor.18a6eaff238fe21f namespace: kserve-ci-e2e-test resourceVersion: "18999" uid: b38d59be-1908-4e9b-a73a-3d4f68600857 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:51:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-2e895d-predictor namespace: kserve-ci-e2e-test resourceVersion: "18994" uid: d724295f-5153-4464-9438-3fb415742e67 kind: Event lastTimestamp: "2026-04-16T18:52:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:51:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:52:09Z" name: isvc-primary-2e895d-predictor.18a6eb02a2546f7e namespace: kserve-ci-e2e-test resourceVersion: "19359" uid: eeee6bf8-8464-4625-85a3-08da08dfea0a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:51:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-2e895d-predictor namespace: kserve-ci-e2e-test resourceVersion: "18994" uid: d724295f-5153-4464-9438-3fb415742e67 kind: Event lastTimestamp: "2026-04-16T18:52:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:51:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:52:09Z" name: isvc-primary-2e895d-predictor.18a6eb02a2baa1dc namespace: kserve-ci-e2e-test resourceVersion: "19360" uid: 45dad302-e024-49b4-9b4f-080c59b2c670 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:52:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-2e895d-predictor namespace: kserve-ci-e2e-test resourceVersion: "19149" uid: d724295f-5153-4464-9438-3fb415742e67 kind: Event lastTimestamp: "2026-04-16T18:52:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:52:54Z" name: isvc-primary-2e895d-predictor.18a6eb0d1f848757 namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: a532bcbf-d00e-4c18-bfd4-bf0bb503df74 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:52:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-2e895d-predictor namespace: kserve-ci-e2e-test resourceVersion: "19149" uid: d724295f-5153-4464-9438-3fb415742e67 kind: Event lastTimestamp: "2026-04-16T18:52:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:52:54Z" name: isvc-primary-2e895d-predictor.18a6eb0d1fec4ebd namespace: kserve-ci-e2e-test resourceVersion: "19751" uid: 539235bb-5ac3-491c-b1d0-0de324bca54c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T18:51:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-2e895d namespace: kserve-ci-e2e-test resourceVersion: "18972" uid: a9837f8a-693c-4243-8be9-5fe501a633d4 kind: Event lastTimestamp: "2026-04-16T18:52:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:52:41Z" name: isvc-primary-2e895d.18a6eaff2360516d namespace: kserve-ci-e2e-test resourceVersion: "19601" uid: 3958e540-48d1-475f-90ce-57ddc3ad2ba5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:51:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-2e895d namespace: kserve-ci-e2e-test resourceVersion: "18972" uid: a9837f8a-693c-4243-8be9-5fe501a633d4 kind: Event lastTimestamp: "2026-04-16T18:51:24Z" message: 'Failed to update status for InferenceService "isvc-primary-2e895d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-2e895d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:51:24Z" name: isvc-primary-2e895d.18a6eaff23c0c323 namespace: kserve-ci-e2e-test resourceVersion: "19015" uid: 67c3b135-e32e-49a4-8c0a-6444438d764c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:51:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-2e895d namespace: kserve-ci-e2e-test resourceVersion: "18972" uid: a9837f8a-693c-4243-8be9-5fe501a633d4 kind: Event lastTimestamp: "2026-04-16T18:51:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-2e895d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:51:24Z" name: isvc-primary-2e895d.18a6eaff23c101be namespace: kserve-ci-e2e-test resourceVersion: "19017" uid: b0583162-cd96-4d0a-bc51-53abac11f911 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:52:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-2e895d namespace: kserve-ci-e2e-test resourceVersion: "19598" uid: a9837f8a-693c-4243-8be9-5fe501a633d4 kind: Event lastTimestamp: "2026-04-16T18:52:41Z" message: InferenceService [isvc-primary-2e895d] is Ready metadata: creationTimestamp: "2026-04-16T18:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:52:41Z" name: isvc-primary-2e895d.18a6eb112702439e namespace: kserve-ci-e2e-test resourceVersion: "19599" uid: ee06b0ec-86e7-4f23-88d3-4633f9e8909a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:42:23.158771Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: be5e84d4-0c1e-4d19-b7af-82f82a9f5f07 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96 to ip-10-0-140-1.ec2.internal metadata: creationTimestamp: "2026-04-16T18:42:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:42:23Z" name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96.18a6ea81146f9c53 namespace: kserve-ci-e2e-test resourceVersion: "14280" uid: 188dc594-dede-44d8-b7b8-d5173a2f2443 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9d7c89bb-dzjdk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:23Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96 namespace: kserve-ci-e2e-test resourceVersion: "14284" uid: be5e84d4-0c1e-4d19-b7af-82f82a9f5f07 kind: Event lastTimestamp: "2026-04-16T18:42:23Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:42:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:42:23Z" name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96.18a6ea812db067f0 namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 5bcb5cec-f471-4dd5-8901-6b93b8db024e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: be5e84d4-0c1e-4d19-b7af-82f82a9f5f07 kind: Event lastTimestamp: "2026-04-16T18:42:23Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" metadata: creationTimestamp: "2026-04-16T18:42:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:42:23Z" name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96.18a6ea812f27bb81 namespace: kserve-ci-e2e-test resourceVersion: "14297" uid: 1d46cb6d-6411-4b0e-8348-00b3bb2489e9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: be5e84d4-0c1e-4d19-b7af-82f82a9f5f07 kind: Event lastTimestamp: "2026-04-16T18:42:26Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" in 3.249s (3.249s including waiting). Image size: 299659693 bytes.' metadata: creationTimestamp: "2026-04-16T18:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:42:26Z" name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96.18a6ea81f0d77a0d namespace: kserve-ci-e2e-test resourceVersion: "14325" uid: 07990aed-5c8e-4593-ac61-960c6c3a9763 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: be5e84d4-0c1e-4d19-b7af-82f82a9f5f07 kind: Event lastTimestamp: "2026-04-16T18:42:26Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:42:26Z" name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96.18a6ea81f3db785f namespace: kserve-ci-e2e-test resourceVersion: "14326" uid: 95954277-59f5-4f4b-b8de-98429255e996 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: be5e84d4-0c1e-4d19-b7af-82f82a9f5f07 kind: Event lastTimestamp: "2026-04-16T18:42:26Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:42:26Z" name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96.18a6ea81f44df30e namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: 7d4018a5-f0f0-4062-9468-cdb3af1a4afe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: be5e84d4-0c1e-4d19-b7af-82f82a9f5f07 kind: Event lastTimestamp: "2026-04-16T18:42:30Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1326" metadata: creationTimestamp: "2026-04-16T18:42:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:42:30Z" name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96.18a6ea82e62259f6 namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 5fb405c3-e709-4ea3-8b7e-1ed7d61ae865 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: be5e84d4-0c1e-4d19-b7af-82f82a9f5f07 kind: Event lastTimestamp: "2026-04-16T18:42:43Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1326" in 12.867s (12.867s including waiting). Image size: 1560613809 bytes.' metadata: creationTimestamp: "2026-04-16T18:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:42:43Z" name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96.18a6ea85e51d8602 namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: f093903c-e056-4eb2-8677-618691a2e3fb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: be5e84d4-0c1e-4d19-b7af-82f82a9f5f07 kind: Event lastTimestamp: "2026-04-16T18:42:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:42:43Z" name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96.18a6ea85e87aad47 namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: 62aa1a28-afe6-4603-927f-954d2ff3a3c1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: be5e84d4-0c1e-4d19-b7af-82f82a9f5f07 kind: Event lastTimestamp: "2026-04-16T18:42:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:42:43Z" name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96.18a6ea85e8ea8e96 namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: ff43668c-c88e-426c-ac4e-947fd5b5ea00 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: be5e84d4-0c1e-4d19-b7af-82f82a9f5f07 kind: Event lastTimestamp: "2026-04-16T18:42:43Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:f2fb11757d99ef46b1388e1609ace6b5f516fdebd4ee81c172186c5de9dc18b9" metadata: creationTimestamp: "2026-04-16T18:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:42:43Z" name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96.18a6ea85e8f767cb namespace: kserve-ci-e2e-test resourceVersion: "14457" uid: ae1f3045-eebc-49e8-acaf-417971766ce5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: be5e84d4-0c1e-4d19-b7af-82f82a9f5f07 kind: Event lastTimestamp: "2026-04-16T18:42:46Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:f2fb11757d99ef46b1388e1609ace6b5f516fdebd4ee81c172186c5de9dc18b9" in 3.019s (3.019s including waiting). Image size: 237894735 bytes.' metadata: creationTimestamp: "2026-04-16T18:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:42:46Z" name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96.18a6ea869ceda4b0 namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 6a6ac471-8eee-407e-9282-ab3844ed67e7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: be5e84d4-0c1e-4d19-b7af-82f82a9f5f07 kind: Event lastTimestamp: "2026-04-16T18:42:46Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-16T18:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:42:46Z" name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96.18a6ea86a046f014 namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: ed13f584-fbd7-448f-81fc-d27dca4a8b4c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: be5e84d4-0c1e-4d19-b7af-82f82a9f5f07 kind: Event lastTimestamp: "2026-04-16T18:42:46Z" message: Started container agent metadata: creationTimestamp: "2026-04-16T18:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:42:46Z" name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96.18a6ea86a0b78437 namespace: kserve-ci-e2e-test resourceVersion: "14479" uid: e0b90440-1927-41ae-9cad-bd3342e68ff0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-16T18:42:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: be5e84d4-0c1e-4d19-b7af-82f82a9f5f07 kind: Event lastTimestamp: "2026-04-16T18:44:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:42:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:44:19Z" name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96.18a6ea86a43a37d9 namespace: kserve-ci-e2e-test resourceVersion: "15280" uid: b9c1158f-d27d-45de-9956-4ca521c69f73 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T18:42:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: be5e84d4-0c1e-4d19-b7af-82f82a9f5f07 kind: Event lastTimestamp: "2026-04-16T18:44:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:42:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:44:19Z" name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96.18a6ea86dffd33cc namespace: kserve-ci-e2e-test resourceVersion: "15281" uid: a34801d1-3ba6-425c-a95b-f778aa1ea0b3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:43:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: be5e84d4-0c1e-4d19-b7af-82f82a9f5f07 kind: Event lastTimestamp: "2026-04-16T18:43:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:43:58Z" name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96.18a6ea973c6d86d5 namespace: kserve-ci-e2e-test resourceVersion: "15040" uid: 0962a68c-9e58-49ed-b7f0-cc86fc00e85e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:43:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: be5e84d4-0c1e-4d19-b7af-82f82a9f5f07 kind: Event lastTimestamp: "2026-04-16T18:43:58Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-16T18:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:43:58Z" name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96.18a6ea973c6ec48d namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: 237fece7-463a-4b3e-9a20-025484677e53 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484 namespace: kserve-ci-e2e-test resourceVersion: "14264" uid: 5d74e068-1797-48b3-9ac5-3e440fd0b05a kind: Event lastTimestamp: "2026-04-16T18:42:23Z" message: 'Created pod: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484-d7w96' metadata: creationTimestamp: "2026-04-16T18:42:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:42:23Z" name: isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484.18a6ea8113d53547 namespace: kserve-ci-e2e-test resourceVersion: "14275" uid: 4987ad11-82a9-4e00-8531-c7700dc0dc46 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-8e196-predictor namespace: kserve-ci-e2e-test resourceVersion: "14263" uid: 7178504e-de69-45f7-87d2-14df05114adb kind: Event lastTimestamp: "2026-04-16T18:42:23Z" message: Scaled up replica set isvc-raw-sklearn-batcher-8e196-predictor-f7dbdb484 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:42:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:42:23Z" name: isvc-raw-sklearn-batcher-8e196-predictor.18a6ea810d683810 namespace: kserve-ci-e2e-test resourceVersion: "14265" uid: 5207d620-51eb-4564-9f9b-3868b566be3d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:42:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-8e196-predictor namespace: kserve-ci-e2e-test resourceVersion: "14272" uid: fdf5d393-f451-4b72-b5f1-0ca4cb026e3d kind: Event lastTimestamp: "2026-04-16T18:43:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:42:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:43:23Z" name: isvc-raw-sklearn-batcher-8e196-predictor.18a6ea8493253e52 namespace: kserve-ci-e2e-test resourceVersion: "14747" uid: 73765c4f-b98a-413c-ad07-aa45d890065b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:42:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-8e196-predictor namespace: kserve-ci-e2e-test resourceVersion: "14272" uid: fdf5d393-f451-4b72-b5f1-0ca4cb026e3d kind: Event lastTimestamp: "2026-04-16T18:43:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:42:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:43:23Z" name: isvc-raw-sklearn-batcher-8e196-predictor.18a6ea849390c988 namespace: kserve-ci-e2e-test resourceVersion: "14748" uid: 7fd565b7-93a0-4328-bfc6-3358025920e3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:43:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-8e196-predictor namespace: kserve-ci-e2e-test resourceVersion: "14413" uid: fdf5d393-f451-4b72-b5f1-0ca4cb026e3d kind: Event lastTimestamp: "2026-04-16T18:43:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:43:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:43:53Z" name: isvc-raw-sklearn-batcher-8e196-predictor.18a6ea8b916ae1a5 namespace: kserve-ci-e2e-test resourceVersion: "14965" uid: d22c9d32-1d33-4e56-b828-65da244b07b6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:43:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-8e196-predictor namespace: kserve-ci-e2e-test resourceVersion: "14413" uid: fdf5d393-f451-4b72-b5f1-0ca4cb026e3d kind: Event lastTimestamp: "2026-04-16T18:43:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:43:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:43:53Z" name: isvc-raw-sklearn-batcher-8e196-predictor.18a6ea8b91cc2a77 namespace: kserve-ci-e2e-test resourceVersion: "14966" uid: 83f50667-fc73-4660-9de8-4dae9841f0bc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-16T18:42:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-8e196 namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: 9bcdfd4d-be6a-487c-a677-a9c33448cde3 kind: Event lastTimestamp: "2026-04-16T18:43:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:42:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:43:49Z" name: isvc-raw-sklearn-batcher-8e196.18a6ea811392ba33 namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: b773fc46-168d-4150-96bb-9c0a64f14fb1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-8e196 namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: 9bcdfd4d-be6a-487c-a677-a9c33448cde3 kind: Event lastTimestamp: "2026-04-16T18:42:23Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-8e196": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-8e196": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:42:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:42:23Z" name: isvc-raw-sklearn-batcher-8e196.18a6ea8113f1cf88 namespace: kserve-ci-e2e-test resourceVersion: "14278" uid: ade50d03-9a8e-41e5-a213-a5bdb95ad121 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-8e196 namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: 9bcdfd4d-be6a-487c-a677-a9c33448cde3 kind: Event lastTimestamp: "2026-04-16T18:42:23Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-8e196": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:42:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:42:23Z" name: isvc-raw-sklearn-batcher-8e196.18a6ea8113f2105e namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 1ae64124-c9ea-41cf-b1ef-415a065a7264 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:43:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-8e196 namespace: kserve-ci-e2e-test resourceVersion: "14938" uid: 9bcdfd4d-be6a-487c-a677-a9c33448cde3 kind: Event lastTimestamp: "2026-04-16T18:43:49Z" message: InferenceService [isvc-raw-sklearn-batcher-8e196] is Ready metadata: creationTimestamp: "2026-04-16T18:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:43:49Z" name: isvc-raw-sklearn-batcher-8e196.18a6ea95198abe8a namespace: kserve-ci-e2e-test resourceVersion: "14939" uid: e9fa0f8a-23f2-4b62-9226-c569268b8fc1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:52:44.796191Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-2e895d-predictor-6b4f9f87-2cczm namespace: kserve-ci-e2e-test resourceVersion: "19645" uid: 107a8f2e-ba63-42a2-94b3-c48d52b96cb3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-2e895d-predictor-6b4f9f87-2cczm to ip-10-0-140-1.ec2.internal metadata: creationTimestamp: "2026-04-16T18:52:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:52:44Z" name: isvc-secondary-2e895d-predictor-6b4f9f87-2cczm.18a6eb11d0e9e9fd namespace: kserve-ci-e2e-test resourceVersion: "19650" uid: 0a22a82c-91af-4d93-b9d8-c761b633e8ce reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9d7c89bb-dzjdk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:52:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-2e895d-predictor-6b4f9f87-2cczm namespace: kserve-ci-e2e-test resourceVersion: "19654" uid: 107a8f2e-ba63-42a2-94b3-c48d52b96cb3 kind: Event lastTimestamp: "2026-04-16T18:52:45Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:52:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:52:45Z" name: isvc-secondary-2e895d-predictor-6b4f9f87-2cczm.18a6eb11ea9a2674 namespace: kserve-ci-e2e-test resourceVersion: "19658" uid: a0808280-05e0-4196-b856-688399be267e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:52:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-2e895d-predictor-6b4f9f87-2cczm namespace: kserve-ci-e2e-test resourceVersion: "19647" uid: 107a8f2e-ba63-42a2-94b3-c48d52b96cb3 kind: Event lastTimestamp: "2026-04-16T18:52:49Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T18:52:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:52:49Z" name: isvc-secondary-2e895d-predictor-6b4f9f87-2cczm.18a6eb11ebe6f748 namespace: kserve-ci-e2e-test resourceVersion: "19699" uid: d7db8fa8-469f-4244-bd30-73453c1fd545 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:52:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-2e895d-predictor-6b4f9f87-2cczm namespace: kserve-ci-e2e-test resourceVersion: "19647" uid: 107a8f2e-ba63-42a2-94b3-c48d52b96cb3 kind: Event lastTimestamp: "2026-04-16T18:52:49Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:52:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:52:49Z" name: isvc-secondary-2e895d-predictor-6b4f9f87-2cczm.18a6eb11ef570542 namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: a8b9efca-f549-4603-a68d-22f8f99a9f67 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:52:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-2e895d-predictor-6b4f9f87-2cczm namespace: kserve-ci-e2e-test resourceVersion: "19647" uid: 107a8f2e-ba63-42a2-94b3-c48d52b96cb3 kind: Event lastTimestamp: "2026-04-16T18:52:49Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:52:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:52:49Z" name: isvc-secondary-2e895d-predictor-6b4f9f87-2cczm.18a6eb11efc3e8c6 namespace: kserve-ci-e2e-test resourceVersion: "19704" uid: 56651a4b-fe99-44ce-82db-431601c4ff59 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:52:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-2e895d-predictor-6b4f9f87-2cczm namespace: kserve-ci-e2e-test resourceVersion: "19647" uid: 107a8f2e-ba63-42a2-94b3-c48d52b96cb3 kind: Event lastTimestamp: "2026-04-16T18:52:53Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-2e895d-predictor-6b4f9f87-2cczm_kserve-ci-e2e-test(107a8f2e-ba63-42a2-94b3-c48d52b96cb3) metadata: creationTimestamp: "2026-04-16T18:52:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:52:53Z" name: isvc-secondary-2e895d-predictor-6b4f9f87-2cczm.18a6eb13c4290b6c namespace: kserve-ci-e2e-test resourceVersion: "19737" uid: 06a3d727-97a5-4841-9f02-270763d3bcb5 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:52:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-2e895d-predictor-6b4f9f87 namespace: kserve-ci-e2e-test resourceVersion: "19636" uid: 1ea4e9a8-1abd-4f28-8239-2767e6154283 kind: Event lastTimestamp: "2026-04-16T18:52:44Z" message: 'Created pod: isvc-secondary-2e895d-predictor-6b4f9f87-2cczm' metadata: creationTimestamp: "2026-04-16T18:52:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:52:44Z" name: isvc-secondary-2e895d-predictor-6b4f9f87.18a6eb11d07bdaab namespace: kserve-ci-e2e-test resourceVersion: "19646" uid: 33503785-3810-4e03-bc4f-38aba5093ca5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:52:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-2e895d-predictor namespace: kserve-ci-e2e-test resourceVersion: "19632" uid: 4b17209a-4d57-4076-9dad-7a2204e248c1 kind: Event lastTimestamp: "2026-04-16T18:52:44Z" message: Scaled up replica set isvc-secondary-2e895d-predictor-6b4f9f87 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:52:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:52:44Z" name: isvc-secondary-2e895d-predictor.18a6eb11cd9026a1 namespace: kserve-ci-e2e-test resourceVersion: "19638" uid: c39f3902-9289-4077-947f-19c61e1d8f2c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:52:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-2e895d-predictor namespace: kserve-ci-e2e-test resourceVersion: "19635" uid: a9cce4a9-4da8-465a-8ed5-da96aba08ac0 kind: Event lastTimestamp: "2026-04-16T18:52:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:52:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:52:59Z" name: isvc-secondary-2e895d-predictor.18a6eb154c7cf7df namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 632fe6ae-a1ee-469b-9461-529222b04278 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:52:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-2e895d-predictor namespace: kserve-ci-e2e-test resourceVersion: "19635" uid: a9cce4a9-4da8-465a-8ed5-da96aba08ac0 kind: Event lastTimestamp: "2026-04-16T18:52:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:52:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:52:59Z" name: isvc-secondary-2e895d-predictor.18a6eb154cd1ea13 namespace: kserve-ci-e2e-test resourceVersion: "19790" uid: fdcb5132-24b8-4891-bcdb-7ebee3b04a42 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T18:52:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-2e895d namespace: kserve-ci-e2e-test resourceVersion: "19625" uid: b8b64b70-56b0-4d1d-a0d5-4c1dd6f75a58 kind: Event lastTimestamp: "2026-04-16T18:52:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:52:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:52:55Z" name: isvc-secondary-2e895d.18a6eb11cd6ef66d namespace: kserve-ci-e2e-test resourceVersion: "19759" uid: fa750f68-2ff8-47e8-84a9-8ccd42618355 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:52:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-2e895d namespace: kserve-ci-e2e-test resourceVersion: "19625" uid: b8b64b70-56b0-4d1d-a0d5-4c1dd6f75a58 kind: Event lastTimestamp: "2026-04-16T18:52:44Z" message: 'Failed to update status for InferenceService "isvc-secondary-2e895d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-2e895d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:52:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:52:44Z" name: isvc-secondary-2e895d.18a6eb11cdc1d6e9 namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: 08f0fa4e-833e-4f17-97fe-4e9dbfdb7daf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:52:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-2e895d namespace: kserve-ci-e2e-test resourceVersion: "19625" uid: b8b64b70-56b0-4d1d-a0d5-4c1dd6f75a58 kind: Event lastTimestamp: "2026-04-16T18:52:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-2e895d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:52:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:52:44Z" name: isvc-secondary-2e895d.18a6eb11cdc2293f namespace: kserve-ci-e2e-test resourceVersion: "19641" uid: c6e1f98e-c43c-4bb4-a4b2-507dde7e7338 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:43:58.388493Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2 namespace: kserve-ci-e2e-test resourceVersion: "15052" uid: 869ab569-4bfe-46d7-947b-59b8f47c7ae1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2 to ip-10-0-140-1.ec2.internal metadata: creationTimestamp: "2026-04-16T18:43:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:43:58Z" name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2.18a6ea974091dfb2 namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: a3e7b7f8-f1a9-4b89-ba16-3bafcb5639e4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9d7c89bb-dzjdk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:43:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2 namespace: kserve-ci-e2e-test resourceVersion: "15071" uid: 869ab569-4bfe-46d7-947b-59b8f47c7ae1 kind: Event lastTimestamp: "2026-04-16T18:43:59Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:43:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:43:59Z" name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2.18a6ea9765fb81d4 namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: aba579a4-b7cf-48e3-8894-c7257cc2a194 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:43:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2 namespace: kserve-ci-e2e-test resourceVersion: "15057" uid: 869ab569-4bfe-46d7-947b-59b8f47c7ae1 kind: Event lastTimestamp: "2026-04-16T18:43:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T18:43:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:43:59Z" name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2.18a6ea97676244c8 namespace: kserve-ci-e2e-test resourceVersion: "15097" uid: db05183e-65f2-4eb2-a958-c967c4085e5a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:43:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2 namespace: kserve-ci-e2e-test resourceVersion: "15057" uid: 869ab569-4bfe-46d7-947b-59b8f47c7ae1 kind: Event lastTimestamp: "2026-04-16T18:43:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:43:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:43:59Z" name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2.18a6ea976a837fac namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: 8957d0b9-5a5b-4263-83cc-f29fc6f400b9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:43:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2 namespace: kserve-ci-e2e-test resourceVersion: "15057" uid: 869ab569-4bfe-46d7-947b-59b8f47c7ae1 kind: Event lastTimestamp: "2026-04-16T18:43:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:43:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:43:59Z" name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2.18a6ea976af2c650 namespace: kserve-ci-e2e-test resourceVersion: "15101" uid: 3ecc9a13-458e-4e92-9e0e-56e58f1d2619 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:44:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2 namespace: kserve-ci-e2e-test resourceVersion: "15057" uid: 869ab569-4bfe-46d7-947b-59b8f47c7ae1 kind: Event lastTimestamp: "2026-04-16T18:44:03Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1326" already present on machine metadata: creationTimestamp: "2026-04-16T18:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:44:03Z" name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2.18a6ea9866b01ceb namespace: kserve-ci-e2e-test resourceVersion: "15146" uid: 19d3d322-1e00-482f-9f43-59c7a538aeda reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:44:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2 namespace: kserve-ci-e2e-test resourceVersion: "15057" uid: 869ab569-4bfe-46d7-947b-59b8f47c7ae1 kind: Event lastTimestamp: "2026-04-16T18:44:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:44:03Z" name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2.18a6ea9869efc631 namespace: kserve-ci-e2e-test resourceVersion: "15150" uid: 144e2da2-b4a1-4e06-9040-528c25509ef3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:44:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2 namespace: kserve-ci-e2e-test resourceVersion: "15057" uid: 869ab569-4bfe-46d7-947b-59b8f47c7ae1 kind: Event lastTimestamp: "2026-04-16T18:44:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:44:03Z" name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2.18a6ea986a607761 namespace: kserve-ci-e2e-test resourceVersion: "15151" uid: 97c262ba-5280-44da-807c-833606352ffb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T18:44:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2 namespace: kserve-ci-e2e-test resourceVersion: "15057" uid: 869ab569-4bfe-46d7-947b-59b8f47c7ae1 kind: Event lastTimestamp: "2026-04-16T18:45:05Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:05Z" name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2.18a6ea98a29317b6 namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: 6a40ce52-8ac4-43cc-9d25-a73277155436 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2 namespace: kserve-ci-e2e-test resourceVersion: "15057" uid: 869ab569-4bfe-46d7-947b-59b8f47c7ae1 kind: Event lastTimestamp: "2026-04-16T18:45:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:38Z" name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2.18a6eaae9946a6b3 namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: f1bbd13d-c4ce-4955-a339-99e9b4ed66d1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:43:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7 namespace: kserve-ci-e2e-test resourceVersion: "15028" uid: 406c0f45-90df-4704-9f1c-01508cd86e1a kind: Event lastTimestamp: "2026-04-16T18:43:58Z" message: 'Created pod: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7-f9gs2' metadata: creationTimestamp: "2026-04-16T18:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:43:58Z" name: isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7.18a6ea97401d8319 namespace: kserve-ci-e2e-test resourceVersion: "15056" uid: 5d894336-953a-4ea0-857b-e62a18529130 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:43:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-1a6d3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15021" uid: ba9f1e82-a5d5-4b53-bd61-71c1f905303b kind: Event lastTimestamp: "2026-04-16T18:43:58Z" message: Scaled up replica set isvc-sklearn-graph-raw-1a6d3-predictor-78bb7647f7 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:43:58Z" name: isvc-sklearn-graph-raw-1a6d3-predictor.18a6ea973b5eef31 namespace: kserve-ci-e2e-test resourceVersion: "15031" uid: 375e7d22-4008-4dff-8f2d-11b0e5721513 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:44:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-1a6d3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15025" uid: 339a6fdd-e5de-4c09-b7b4-fdf0399ff5bf kind: Event lastTimestamp: "2026-04-16T18:44:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:44:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:44:13Z" name: isvc-sklearn-graph-raw-1a6d3-predictor.18a6ea9ab9d9edcf namespace: kserve-ci-e2e-test resourceVersion: "15233" uid: 36a4b523-9418-4b4f-8f0b-3e4adbc69873 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:44:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-1a6d3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15025" uid: 339a6fdd-e5de-4c09-b7b4-fdf0399ff5bf kind: Event lastTimestamp: "2026-04-16T18:44:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:44:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:44:13Z" name: isvc-sklearn-graph-raw-1a6d3-predictor.18a6ea9aba46a4f7 namespace: kserve-ci-e2e-test resourceVersion: "15235" uid: c9a8055a-56c8-442d-8be4-8d662f4750b0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:44:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-1a6d3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15234" uid: 339a6fdd-e5de-4c09-b7b4-fdf0399ff5bf kind: Event lastTimestamp: "2026-04-16T18:45:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:44:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:45:28Z" name: isvc-sklearn-graph-raw-1a6d3-predictor.18a6ea9e396bb292 namespace: kserve-ci-e2e-test resourceVersion: "15833" uid: 6085de8c-a2ac-4b21-89eb-c5de1727df4b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:44:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-1a6d3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15234" uid: 339a6fdd-e5de-4c09-b7b4-fdf0399ff5bf kind: Event lastTimestamp: "2026-04-16T18:45:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:44:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:45:28Z" name: isvc-sklearn-graph-raw-1a6d3-predictor.18a6ea9e39d6d250 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: e4cf249c-4aeb-4401-a63f-75dae24b4f43 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T18:43:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-1a6d3 namespace: kserve-ci-e2e-test resourceVersion: "15008" uid: c06cfe69-e3d1-4bf4-986b-6c99083cf1bc kind: Event lastTimestamp: "2026-04-16T18:45:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:45:15Z" name: isvc-sklearn-graph-raw-1a6d3.18a6ea973b2bcbc8 namespace: kserve-ci-e2e-test resourceVersion: "15706" uid: 358dcdb5-a4d5-4fdc-9913-fa92e8854967 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:43:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-1a6d3 namespace: kserve-ci-e2e-test resourceVersion: "15008" uid: c06cfe69-e3d1-4bf4-986b-6c99083cf1bc kind: Event lastTimestamp: "2026-04-16T18:43:58Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-1a6d3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-1a6d3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:43:58Z" name: isvc-sklearn-graph-raw-1a6d3.18a6ea973bb4d952 namespace: kserve-ci-e2e-test resourceVersion: "15034" uid: e1cab72e-a2cc-4b4b-aac2-cc7b3ddf2bf8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-1a6d3 namespace: kserve-ci-e2e-test resourceVersion: "15704" uid: c06cfe69-e3d1-4bf4-986b-6c99083cf1bc kind: Event lastTimestamp: "2026-04-16T18:45:15Z" message: InferenceService [isvc-sklearn-graph-raw-1a6d3] is Ready metadata: creationTimestamp: "2026-04-16T18:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:45:15Z" name: isvc-sklearn-graph-raw-1a6d3.18a6eaa92ed01f02 namespace: kserve-ci-e2e-test resourceVersion: "15705" uid: 9d72995c-a19b-4beb-9d61-6833f09d5fe0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:45:38.707972Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv namespace: kserve-ci-e2e-test resourceVersion: "16010" uid: aec133a5-e6b1-4385-b8a0-386b10bbb6ab kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv to ip-10-0-140-1.ec2.internal metadata: creationTimestamp: "2026-04-16T18:45:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:45:38Z" name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv.18a6eaae9c139820 namespace: kserve-ci-e2e-test resourceVersion: "16021" uid: d2f677af-3a1b-4119-8065-86e5b841213c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9d7c89bb-dzjdk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv namespace: kserve-ci-e2e-test resourceVersion: "16025" uid: aec133a5-e6b1-4385-b8a0-386b10bbb6ab kind: Event lastTimestamp: "2026-04-16T18:45:39Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:45:39Z" name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv.18a6eaaeb560c9c4 namespace: kserve-ci-e2e-test resourceVersion: "16053" uid: d5fae9aa-29bb-4df0-96d6-2e54b455d0fe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: aec133a5-e6b1-4385-b8a0-386b10bbb6ab kind: Event lastTimestamp: "2026-04-16T18:45:39Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T18:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:39Z" name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv.18a6eaaeb6b4e852 namespace: kserve-ci-e2e-test resourceVersion: "16055" uid: 5d5a5aa2-d73d-4a2b-a702-08e49937d23b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: aec133a5-e6b1-4385-b8a0-386b10bbb6ab kind: Event lastTimestamp: "2026-04-16T18:45:39Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:39Z" name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv.18a6eaaeb989ef91 namespace: kserve-ci-e2e-test resourceVersion: "16059" uid: 5ef44476-5ed0-417e-9dd7-109ca6d5c5d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: aec133a5-e6b1-4385-b8a0-386b10bbb6ab kind: Event lastTimestamp: "2026-04-16T18:45:39Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:39Z" name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv.18a6eaaeb9f8c650 namespace: kserve-ci-e2e-test resourceVersion: "16060" uid: 73af76a7-c1ae-4d10-a28d-127c54ff7a68 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: aec133a5-e6b1-4385-b8a0-386b10bbb6ab kind: Event lastTimestamp: "2026-04-16T18:45:43Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1326" already present on machine metadata: creationTimestamp: "2026-04-16T18:45:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:43Z" name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv.18a6eaafc4056d3b namespace: kserve-ci-e2e-test resourceVersion: "16116" uid: 0bd42b03-c217-4f6d-bb2f-4794f7c712bf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: aec133a5-e6b1-4385-b8a0-386b10bbb6ab kind: Event lastTimestamp: "2026-04-16T18:45:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:45:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:43Z" name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv.18a6eaafc77e2e15 namespace: kserve-ci-e2e-test resourceVersion: "16122" uid: 717a7f26-f74e-473a-8f5a-8be726ec84bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: aec133a5-e6b1-4385-b8a0-386b10bbb6ab kind: Event lastTimestamp: "2026-04-16T18:45:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:45:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:43Z" name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv.18a6eaafc7f07101 namespace: kserve-ci-e2e-test resourceVersion: "16123" uid: 7de69ea6-1a27-4c41-9d81-8a5794b96c97 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T18:45:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: aec133a5-e6b1-4385-b8a0-386b10bbb6ab kind: Event lastTimestamp: "2026-04-16T18:46:46Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:45:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:46:46Z" name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv.18a6eab0001444ac namespace: kserve-ci-e2e-test resourceVersion: "16596" uid: f0838b88-fe7a-4514-82fa-14d16ea238a6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: aec133a5-e6b1-4385-b8a0-386b10bbb6ab kind: Event lastTimestamp: "2026-04-16T18:47:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:47:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:19Z" name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv.18a6eac5fa9811e5 namespace: kserve-ci-e2e-test resourceVersion: "16957" uid: 19556d58-85bb-4e6b-b988-5118bcf670bc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696 namespace: kserve-ci-e2e-test resourceVersion: "15977" uid: 33351d70-6d3c-4960-a521-d254d873a670 kind: Event lastTimestamp: "2026-04-16T18:45:38Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696-vj9fv' metadata: creationTimestamp: "2026-04-16T18:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:45:38Z" name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696.18a6eaae9ba1275b namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: 3b4f973d-03b4-4416-a8d9-e7da60a05f35 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 2cf45fc4-ad95-4c63-bf59-21998f23c1d5 kind: Event lastTimestamp: "2026-04-16T18:45:38Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-fd0ec-predictor-5b56948696 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:45:38Z" name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor.18a6eaae96d34dd8 namespace: kserve-ci-e2e-test resourceVersion: "15979" uid: 459c7585-4057-4f1d-91ef-27b42523cab0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:45:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: 659be5f4-bd20-475d-b0d7-23080e1bd598 kind: Event lastTimestamp: "2026-04-16T18:46:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:46:23Z" name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor.18a6eab21553ece0 namespace: kserve-ci-e2e-test resourceVersion: "16419" uid: 85a162f1-cd1d-4da3-9008-99807d09fce5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:45:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: 659be5f4-bd20-475d-b0d7-23080e1bd598 kind: Event lastTimestamp: "2026-04-16T18:46:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:46:23Z" name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor.18a6eab215d53630 namespace: kserve-ci-e2e-test resourceVersion: "16420" uid: c3c1905d-04b6-4db7-a9de-8bfb3a5e440e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:46:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 659be5f4-bd20-475d-b0d7-23080e1bd598 kind: Event lastTimestamp: "2026-04-16T18:47:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:46:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:47:08Z" name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor.18a6eabc932b155a namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: ade90296-c61c-4d69-8905-26b69ad9e93b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:46:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 659be5f4-bd20-475d-b0d7-23080e1bd598 kind: Event lastTimestamp: "2026-04-16T18:47:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:46:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:47:08Z" name: isvc-sklearn-graph-raw-hpa-fd0ec-predictor.18a6eabc939b8b21 namespace: kserve-ci-e2e-test resourceVersion: "16769" uid: 4a06afd0-3e44-430a-b022-7fccb9fe2234 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T18:45:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-fd0ec namespace: kserve-ci-e2e-test resourceVersion: "15955" uid: fb906ba0-bd2c-4ebc-af3d-fd244b917ec5 kind: Event lastTimestamp: "2026-04-16T18:46:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:46:56Z" name: isvc-sklearn-graph-raw-hpa-fd0ec.18a6eaae96af6d13 namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: c008fec8-8378-4764-beb5-981edd0f1eef reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:46:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-fd0ec namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: fb906ba0-bd2c-4ebc-af3d-fd244b917ec5 kind: Event lastTimestamp: "2026-04-16T18:46:56Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-fd0ec] is Ready metadata: creationTimestamp: "2026-04-16T18:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:46:56Z" name: isvc-sklearn-graph-raw-hpa-fd0ec.18a6eac0c480f4d9 namespace: kserve-ci-e2e-test resourceVersion: "16671" uid: 8befb7f4-56dc-4b46-a0eb-36d2345b1cbf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:48:54.427831Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k namespace: kserve-ci-e2e-test resourceVersion: "17776" uid: 6c2e06ef-387f-4446-b5cf-1edb2fae7e01 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k to ip-10-0-140-1.ec2.internal metadata: creationTimestamp: "2026-04-16T18:48:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:48:54Z" name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k.18a6eadc2de449dd namespace: kserve-ci-e2e-test resourceVersion: "17783" uid: e0ccaa5e-9381-4290-8000-8a64585e38a9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9d7c89bb-dzjdk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:54Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k namespace: kserve-ci-e2e-test resourceVersion: "17790" uid: 6c2e06ef-387f-4446-b5cf-1edb2fae7e01 kind: Event lastTimestamp: "2026-04-16T18:48:54Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:48:54Z" name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k.18a6eadc46ea56e2 namespace: kserve-ci-e2e-test resourceVersion: "17809" uid: 344b016d-2a36-471a-85d5-06824ed70a53 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k namespace: kserve-ci-e2e-test resourceVersion: "17778" uid: 6c2e06ef-387f-4446-b5cf-1edb2fae7e01 kind: Event lastTimestamp: "2026-04-16T18:48:54Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T18:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:48:54Z" name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k.18a6eadc48fc4dab namespace: kserve-ci-e2e-test resourceVersion: "17812" uid: ff6aac53-b9f8-4cfa-bcf4-8de114c27793 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k namespace: kserve-ci-e2e-test resourceVersion: "17778" uid: 6c2e06ef-387f-4446-b5cf-1edb2fae7e01 kind: Event lastTimestamp: "2026-04-16T18:48:54Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:48:54Z" name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k.18a6eadc4cc6f0bc namespace: kserve-ci-e2e-test resourceVersion: "17815" uid: d2cb894d-9d2f-48c1-afdb-82ee001a5cd3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k namespace: kserve-ci-e2e-test resourceVersion: "17778" uid: 6c2e06ef-387f-4446-b5cf-1edb2fae7e01 kind: Event lastTimestamp: "2026-04-16T18:48:54Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:48:54Z" name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k.18a6eadc4d355e46 namespace: kserve-ci-e2e-test resourceVersion: "17816" uid: de443f0b-eecf-4b2b-80d1-eb77144d6bce reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k namespace: kserve-ci-e2e-test resourceVersion: "17778" uid: 6c2e06ef-387f-4446-b5cf-1edb2fae7e01 kind: Event lastTimestamp: "2026-04-16T18:48:59Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1326" already present on machine metadata: creationTimestamp: "2026-04-16T18:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:48:59Z" name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k.18a6eadd5360a2a2 namespace: kserve-ci-e2e-test resourceVersion: "17858" uid: 613aa2b1-927d-4284-a980-b47c59eca318 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k namespace: kserve-ci-e2e-test resourceVersion: "17778" uid: 6c2e06ef-387f-4446-b5cf-1edb2fae7e01 kind: Event lastTimestamp: "2026-04-16T18:48:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:48:59Z" name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k.18a6eadd56a3c6b6 namespace: kserve-ci-e2e-test resourceVersion: "17862" uid: 28ca7672-9457-4049-9733-ab9f70650c1c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k namespace: kserve-ci-e2e-test resourceVersion: "17778" uid: 6c2e06ef-387f-4446-b5cf-1edb2fae7e01 kind: Event lastTimestamp: "2026-04-16T18:48:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:48:59Z" name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k.18a6eadd5716d225 namespace: kserve-ci-e2e-test resourceVersion: "17863" uid: a01728bd-7a38-44d3-838b-43578094984a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 17 eventTime: null firstTimestamp: "2026-04-16T18:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k namespace: kserve-ci-e2e-test resourceVersion: "17778" uid: 6c2e06ef-387f-4446-b5cf-1edb2fae7e01 kind: Event lastTimestamp: "2026-04-16T18:51:27Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:51:27Z" name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k.18a6eadd8f48ad06 namespace: kserve-ci-e2e-test resourceVersion: "19055" uid: 660d6225-3200-4bd2-abcd-412845b8bbf2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k namespace: kserve-ci-e2e-test resourceVersion: "17778" uid: 6c2e06ef-387f-4446-b5cf-1edb2fae7e01 kind: Event lastTimestamp: "2026-04-16T18:51:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:51:24Z" name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k.18a6eaff23bffe89 namespace: kserve-ci-e2e-test resourceVersion: "19001" uid: 6aeb91df-712a-4af9-af6e-abb88c58e690 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7 namespace: kserve-ci-e2e-test resourceVersion: "17754" uid: f03f5b0f-7a93-4956-8dad-78d339e008f3 kind: Event lastTimestamp: "2026-04-16T18:48:54Z" message: 'Created pod: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7-hxp4k' metadata: creationTimestamp: "2026-04-16T18:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:48:54Z" name: isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7.18a6eadc2d57168a namespace: kserve-ci-e2e-test resourceVersion: "17781" uid: 41ba3f2c-9f52-4438-9e0d-d35ac03a5a2a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-858f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "17750" uid: 67f51bae-cbc2-445d-aa61-aaca1d456f0b kind: Event lastTimestamp: "2026-04-16T18:48:54Z" message: Scaled up replica set isvc-sklearn-scale-raw-858f6-predictor-58964bb9c7 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:48:54Z" name: isvc-sklearn-scale-raw-858f6-predictor.18a6eadc28ac2686 namespace: kserve-ci-e2e-test resourceVersion: "17758" uid: c9ced392-91c2-462e-80d2-a5d19ef9b211 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:49:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-858f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "17753" uid: 54e8d0a5-d66d-40c9-9927-5910684e6488 kind: Event lastTimestamp: "2026-04-16T18:49:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:49:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:39Z" name: isvc-sklearn-scale-raw-858f6-predictor.18a6eadfa7c9c570 namespace: kserve-ci-e2e-test resourceVersion: "18214" uid: e1ccd95e-b6dd-4511-a271-460da3ec1cb9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:49:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-858f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "17753" uid: 54e8d0a5-d66d-40c9-9927-5910684e6488 kind: Event lastTimestamp: "2026-04-16T18:49:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:49:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:39Z" name: isvc-sklearn-scale-raw-858f6-predictor.18a6eadfa830f544 namespace: kserve-ci-e2e-test resourceVersion: "18215" uid: a66998e1-c57c-4320-9d3d-b500c31c359d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:49:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-858f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "17938" uid: 54e8d0a5-d66d-40c9-9927-5910684e6488 kind: Event lastTimestamp: "2026-04-16T18:51:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:49:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:51:24Z" name: isvc-sklearn-scale-raw-858f6-predictor.18a6eaea253110d5 namespace: kserve-ci-e2e-test resourceVersion: "18970" uid: 6ec59835-58dc-44f2-91c1-22fcafd07015 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:49:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-858f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "17938" uid: 54e8d0a5-d66d-40c9-9927-5910684e6488 kind: Event lastTimestamp: "2026-04-16T18:51:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:49:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:51:24Z" name: isvc-sklearn-scale-raw-858f6-predictor.18a6eaea259fe437 namespace: kserve-ci-e2e-test resourceVersion: "18971" uid: a88bdac9-2aca-4ed1-8ea6-256c0679152d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T18:48:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-858f6 namespace: kserve-ci-e2e-test resourceVersion: "17741" uid: 43bbe6fe-59b3-434c-8be5-45e552078b51 kind: Event lastTimestamp: "2026-04-16T18:51:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:51:17Z" name: isvc-sklearn-scale-raw-858f6.18a6eadc28b26820 namespace: kserve-ci-e2e-test resourceVersion: "18921" uid: 92afb75c-129f-482d-8801-a0a1ccae19d7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-858f6 namespace: kserve-ci-e2e-test resourceVersion: "17760" uid: 43bbe6fe-59b3-434c-8be5-45e552078b51 kind: Event lastTimestamp: "2026-04-16T18:48:54Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-858f6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-858f6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:48:54Z" name: isvc-sklearn-scale-raw-858f6.18a6eadc2cfe5b2b namespace: kserve-ci-e2e-test resourceVersion: "17777" uid: 05c70ce4-3a2f-42b5-97b2-a15f1b3bd897 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:51:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-858f6 namespace: kserve-ci-e2e-test resourceVersion: "18918" uid: 43bbe6fe-59b3-434c-8be5-45e552078b51 kind: Event lastTimestamp: "2026-04-16T18:51:17Z" message: InferenceService [isvc-sklearn-scale-raw-858f6] is Ready metadata: creationTimestamp: "2026-04-16T18:51:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:51:17Z" name: isvc-sklearn-scale-raw-858f6.18a6eafd88738e67 namespace: kserve-ci-e2e-test resourceVersion: "18919" uid: 91325afc-4ade-4857-91d3-0987f8164337 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:43:58.454691Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr namespace: kserve-ci-e2e-test resourceVersion: "15073" uid: 90c5376f-d100-46e2-ad46-05e58f34b6af kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr to ip-10-0-142-225.ec2.internal metadata: creationTimestamp: "2026-04-16T18:43:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:43:58Z" name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr.18a6ea9744840858 namespace: kserve-ci-e2e-test resourceVersion: "15080" uid: d19a01bf-e476-40f8-b859-511173ead546 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9d7c89bb-dzjdk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:43:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr namespace: kserve-ci-e2e-test resourceVersion: "15084" uid: 90c5376f-d100-46e2-ad46-05e58f34b6af kind: Event lastTimestamp: "2026-04-16T18:43:58Z" message: Add eth0 [10.132.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:43:58Z" name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr.18a6ea975cef52a5 namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: 0d521b68-5043-4114-b821-63c42bb775d6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:43:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 90c5376f-d100-46e2-ad46-05e58f34b6af kind: Event lastTimestamp: "2026-04-16T18:43:58Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" metadata: creationTimestamp: "2026-04-16T18:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:43:58Z" name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr.18a6ea975e480c8c namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: cb725ee7-90af-4e3b-82e5-b3cb7fb2186d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:44:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 90c5376f-d100-46e2-ad46-05e58f34b6af kind: Event lastTimestamp: "2026-04-16T18:44:01Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" in 3.043s (3.043s including waiting). Image size: 299659693 bytes.' metadata: creationTimestamp: "2026-04-16T18:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:44:01Z" name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr.18a6ea9813b71895 namespace: kserve-ci-e2e-test resourceVersion: "15131" uid: 75b6b77c-feb9-439c-a39b-76a77da1c7cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:44:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 90c5376f-d100-46e2-ad46-05e58f34b6af kind: Event lastTimestamp: "2026-04-16T18:44:01Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:44:01Z" name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr.18a6ea98168f2a78 namespace: kserve-ci-e2e-test resourceVersion: "15132" uid: 47087783-7c87-428e-a823-d8c404895337 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:44:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 90c5376f-d100-46e2-ad46-05e58f34b6af kind: Event lastTimestamp: "2026-04-16T18:44:01Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:44:01Z" name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr.18a6ea981744fa35 namespace: kserve-ci-e2e-test resourceVersion: "15133" uid: f39f5766-239c-4c5f-9f02-d1bc204dea3b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:44:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 90c5376f-d100-46e2-ad46-05e58f34b6af kind: Event lastTimestamp: "2026-04-16T18:44:06Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-16T18:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:44:06Z" name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr.18a6ea99339d9eb1 namespace: kserve-ci-e2e-test resourceVersion: "15182" uid: 59923f41-29da-4371-ab57-bf6bba25be13 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:44:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 90c5376f-d100-46e2-ad46-05e58f34b6af kind: Event lastTimestamp: "2026-04-16T18:44:24Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 17.917s (17.917s including waiting). Image size: 1306229499 bytes.' metadata: creationTimestamp: "2026-04-16T18:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:44:24Z" name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr.18a6ea9d5f9844bc namespace: kserve-ci-e2e-test resourceVersion: "15321" uid: 087f3800-cef1-42e9-baeb-685dec4b036c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:44:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 90c5376f-d100-46e2-ad46-05e58f34b6af kind: Event lastTimestamp: "2026-04-16T18:44:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:44:24Z" name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr.18a6ea9d62705328 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: c89386a7-7f40-4f25-8b48-94807e7750eb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:44:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 90c5376f-d100-46e2-ad46-05e58f34b6af kind: Event lastTimestamp: "2026-04-16T18:44:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:44:24Z" name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr.18a6ea9d62ddb70d namespace: kserve-ci-e2e-test resourceVersion: "15324" uid: 49aa2062-c2d2-4feb-827a-0a9c8b6db8ac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:44:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 90c5376f-d100-46e2-ad46-05e58f34b6af kind: Event lastTimestamp: "2026-04-16T18:45:15Z" message: 'Readiness probe failed: dial tcp 10.132.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:15Z" name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr.18a6ea9d67c0b92d namespace: kserve-ci-e2e-test resourceVersion: "15709" uid: 68376c74-75e2-4323-8958-6fec6b05d887 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 90c5376f-d100-46e2-ad46-05e58f34b6af kind: Event lastTimestamp: "2026-04-16T18:45:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:38Z" name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr.18a6eaae967bb387 namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: fe9459e9-820d-4448-8a23-bba7e5f7c0e7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:43:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44 namespace: kserve-ci-e2e-test resourceVersion: "15055" uid: cb5dab95-090d-41c2-a4e4-9cd2312a74ea kind: Event lastTimestamp: "2026-04-16T18:43:58Z" message: 'Created pod: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44-2vczr' metadata: creationTimestamp: "2026-04-16T18:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:43:58Z" name: isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44.18a6ea9744133c09 namespace: kserve-ci-e2e-test resourceVersion: "15076" uid: ab40c2b8-c3d1-413c-ba10-afa0757e893b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:43:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-1a6d3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: cb84f85a-9689-4b40-8a2e-b643af8c9188 kind: Event lastTimestamp: "2026-04-16T18:43:58Z" message: Scaled up replica set isvc-xgboost-graph-raw-1a6d3-predictor-6ddf8d4b44 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:43:58Z" name: isvc-xgboost-graph-raw-1a6d3-predictor.18a6ea9740a16ae1 namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: 5e4b6adb-2ee4-40e6-a46a-c6145beb1a3b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:44:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-1a6d3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: c4ace44d-ee86-4c12-86ad-26ebf90281ad kind: Event lastTimestamp: "2026-04-16T18:44:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:44:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:44:28Z" name: isvc-xgboost-graph-raw-1a6d3-predictor.18a6ea9abf505749 namespace: kserve-ci-e2e-test resourceVersion: "15361" uid: 15ac68ec-dd0d-4098-93e2-4259119f1267 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:44:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-1a6d3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: c4ace44d-ee86-4c12-86ad-26ebf90281ad kind: Event lastTimestamp: "2026-04-16T18:44:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:44:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:44:28Z" name: isvc-xgboost-graph-raw-1a6d3-predictor.18a6ea9abfbe25d6 namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: fbc3a2da-1c6e-42ff-bf7f-9278de45fd8e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:44:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-1a6d3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: c4ace44d-ee86-4c12-86ad-26ebf90281ad kind: Event lastTimestamp: "2026-04-16T18:45:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:45:28Z" name: isvc-xgboost-graph-raw-1a6d3-predictor.18a6eaa1bd41f060 namespace: kserve-ci-e2e-test resourceVersion: "15845" uid: 1f8dd99b-2673-4a28-a81f-3e7581dfd734 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:44:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-1a6d3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: c4ace44d-ee86-4c12-86ad-26ebf90281ad kind: Event lastTimestamp: "2026-04-16T18:45:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:45:28Z" name: isvc-xgboost-graph-raw-1a6d3-predictor.18a6eaa1bdb45d65 namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: f566abea-fbf1-4273-8186-a7900458e4c2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T18:43:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-1a6d3 namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: 13be92d6-0670-4adc-a720-1fc318ce75de kind: Event lastTimestamp: "2026-04-16T18:45:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:45:25Z" name: isvc-xgboost-graph-raw-1a6d3.18a6ea9740954cbe namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: 63e4bc67-97a9-4443-b4e3-276bd5ce1043 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:43:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-1a6d3 namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: 13be92d6-0670-4adc-a720-1fc318ce75de kind: Event lastTimestamp: "2026-04-16T18:43:58Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-1a6d3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-1a6d3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:43:58Z" name: isvc-xgboost-graph-raw-1a6d3.18a6ea97413c7049 namespace: kserve-ci-e2e-test resourceVersion: "15068" uid: 91489baf-f696-4851-9478-2baefb4f839f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-1a6d3 namespace: kserve-ci-e2e-test resourceVersion: "15791" uid: 13be92d6-0670-4adc-a720-1fc318ce75de kind: Event lastTimestamp: "2026-04-16T18:45:25Z" message: InferenceService [isvc-xgboost-graph-raw-1a6d3] is Ready metadata: creationTimestamp: "2026-04-16T18:45:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:45:25Z" name: isvc-xgboost-graph-raw-1a6d3.18a6eaaba0ab330a namespace: kserve-ci-e2e-test resourceVersion: "15792" uid: f790b000-bba6-4d03-990f-d819e40fc035 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:45:38.752861Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 0e5aea4b-904f-41e8-90e8-24605d00d302 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p to ip-10-0-142-225.ec2.internal metadata: creationTimestamp: "2026-04-16T18:45:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:45:38Z" name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p.18a6eaae9ec08a73 namespace: kserve-ci-e2e-test resourceVersion: "16036" uid: b4892b30-fbb0-443a-b9a5-3d4d4e86d81a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9d7c89bb-dzjdk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 0e5aea4b-904f-41e8-90e8-24605d00d302 kind: Event lastTimestamp: "2026-04-16T18:45:39Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:45:39Z" name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p.18a6eaaeb775e544 namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: ba2671fb-3a3c-4ff9-b5c2-cf6a949f5bfb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 0e5aea4b-904f-41e8-90e8-24605d00d302 kind: Event lastTimestamp: "2026-04-16T18:45:39Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T18:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:39Z" name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p.18a6eaaeb8dc4de2 namespace: kserve-ci-e2e-test resourceVersion: "16058" uid: 967d9015-17cb-4595-aa5d-7521aac1bb92 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 0e5aea4b-904f-41e8-90e8-24605d00d302 kind: Event lastTimestamp: "2026-04-16T18:45:39Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:39Z" name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p.18a6eaaebbb3d478 namespace: kserve-ci-e2e-test resourceVersion: "16062" uid: 0d447fcf-346b-4d5c-9cf4-79442e611dd8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 0e5aea4b-904f-41e8-90e8-24605d00d302 kind: Event lastTimestamp: "2026-04-16T18:45:39Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:39Z" name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p.18a6eaaebc1c07d2 namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 35a0f30f-d775-4292-8e5b-8b9476bd1dee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 0e5aea4b-904f-41e8-90e8-24605d00d302 kind: Event lastTimestamp: "2026-04-16T18:45:43Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-16T18:45:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:43Z" name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p.18a6eaaf9d9dcaf1 namespace: kserve-ci-e2e-test resourceVersion: "16105" uid: 253659c8-43c6-49fd-a918-1da6a088a781 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 0e5aea4b-904f-41e8-90e8-24605d00d302 kind: Event lastTimestamp: "2026-04-16T18:45:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:45:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:43Z" name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p.18a6eaafa10458dc namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 38774d23-4ef0-4f83-a217-1db0c8879369 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 0e5aea4b-904f-41e8-90e8-24605d00d302 kind: Event lastTimestamp: "2026-04-16T18:45:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:45:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:43Z" name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p.18a6eaafa1763fd4 namespace: kserve-ci-e2e-test resourceVersion: "16112" uid: 0792267f-802c-499b-9e7d-ba6948b9468b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:45:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 0e5aea4b-904f-41e8-90e8-24605d00d302 kind: Event lastTimestamp: "2026-04-16T18:46:35Z" message: 'Readiness probe failed: dial tcp 10.132.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:45:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:46:35Z" name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p.18a6eaafd9891c74 namespace: kserve-ci-e2e-test resourceVersion: "16499" uid: 3a127fa8-e761-4fd0-b953-a4b1481eebfb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 0e5aea4b-904f-41e8-90e8-24605d00d302 kind: Event lastTimestamp: "2026-04-16T18:47:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:18Z" name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p.18a6eac5f443314e namespace: kserve-ci-e2e-test resourceVersion: "16923" uid: a5a30779-0f16-4aa0-adb4-a620fd1b38cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945 namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: 1f0bda67-aeff-443a-a8c5-a78c1981611f kind: Event lastTimestamp: "2026-04-16T18:45:38Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945-xdm6p' metadata: creationTimestamp: "2026-04-16T18:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:45:38Z" name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945.18a6eaae9e4e6f77 namespace: kserve-ci-e2e-test resourceVersion: "16033" uid: 0f33402b-c5a4-4179-9467-312a61b54190 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "16007" uid: d8680c28-bdd3-4140-9272-b4f0b60aebbd kind: Event lastTimestamp: "2026-04-16T18:45:38Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-fd0ec-predictor-748987f945 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:45:38Z" name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor.18a6eaae9beb099b namespace: kserve-ci-e2e-test resourceVersion: "16019" uid: 7123cf7d-67f9-45a3-8f64-dcba49f2fc8d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:45:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "16011" uid: 4e48effc-c907-4ee6-898b-2be8d7ae519a kind: Event lastTimestamp: "2026-04-16T18:46:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:46:23Z" name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor.18a6eab21a746e75 namespace: kserve-ci-e2e-test resourceVersion: "16422" uid: 54aaa2c4-c836-4edb-b4eb-8f84620775c0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:45:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "16011" uid: 4e48effc-c907-4ee6-898b-2be8d7ae519a kind: Event lastTimestamp: "2026-04-16T18:46:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:46:23Z" name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor.18a6eab21ada17d4 namespace: kserve-ci-e2e-test resourceVersion: "16423" uid: 25b53d43-33f1-44c4-afb8-2fc18ee7bfdd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:46:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: 4e48effc-c907-4ee6-898b-2be8d7ae519a kind: Event lastTimestamp: "2026-04-16T18:47:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:46:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:47:08Z" name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor.18a6eabc97056e6d namespace: kserve-ci-e2e-test resourceVersion: "16781" uid: a8aa7d48-dfdb-4626-9b8a-4b755d7c6f2a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:46:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: 4e48effc-c907-4ee6-898b-2be8d7ae519a kind: Event lastTimestamp: "2026-04-16T18:47:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:46:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:47:08Z" name: isvc-xgboost-graph-raw-hpa-fd0ec-predictor.18a6eabc9770ba29 namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: d3b3d6ad-3be8-49ec-8bc5-dfb7c9bcb3ee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T18:45:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-fd0ec namespace: kserve-ci-e2e-test resourceVersion: "15998" uid: 5c961b25-e384-47fb-b49e-8796edfa9430 kind: Event lastTimestamp: "2026-04-16T18:46:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:46:45Z" name: isvc-xgboost-graph-raw-hpa-fd0ec.18a6eaae9ba8c6ef namespace: kserve-ci-e2e-test resourceVersion: "16582" uid: 41a77aac-ac9c-4fac-b468-db593e9b50cc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:46:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-fd0ec namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: 5c961b25-e384-47fb-b49e-8796edfa9430 kind: Event lastTimestamp: "2026-04-16T18:46:45Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-fd0ec] is Ready metadata: creationTimestamp: "2026-04-16T18:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:46:45Z" name: isvc-xgboost-graph-raw-hpa-fd0ec.18a6eabe12054697 namespace: kserve-ci-e2e-test resourceVersion: "16580" uid: 7948e6f5-e62a-4775-b222-06c6e51515ec reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:47:19.025160Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-79732-predictor-77bcc96986-82s7n namespace: kserve-ci-e2e-test resourceVersion: "16931" uid: e3ffb519-60f9-4431-baf6-36792845e4b2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-79732-predictor-77bcc96986-82s7n to ip-10-0-142-225.ec2.internal metadata: creationTimestamp: "2026-04-16T18:47:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:47:19Z" name: message-dumper-raw-79732-predictor-77bcc96986-82s7n.18a6eac5f7727856 namespace: kserve-ci-e2e-test resourceVersion: "16935" uid: 574b637d-35b1-4129-8889-34b7d326df04 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9d7c89bb-dzjdk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:19Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-79732-predictor-77bcc96986-82s7n namespace: kserve-ci-e2e-test resourceVersion: "16944" uid: e3ffb519-60f9-4431-baf6-36792845e4b2 kind: Event lastTimestamp: "2026-04-16T18:47:19Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:47:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:47:19Z" name: message-dumper-raw-79732-predictor-77bcc96986-82s7n.18a6eac5fe8f7062 namespace: kserve-ci-e2e-test resourceVersion: "16960" uid: dbb3774e-0241-4be8-87fa-a6e7267faa8e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-79732-predictor-77bcc96986-82s7n namespace: kserve-ci-e2e-test resourceVersion: "16932" uid: e3ffb519-60f9-4431-baf6-36792845e4b2 kind: Event lastTimestamp: "2026-04-16T18:47:19Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-16T18:47:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:19Z" name: message-dumper-raw-79732-predictor-77bcc96986-82s7n.18a6eac5ffd7ef8c namespace: kserve-ci-e2e-test resourceVersion: "16962" uid: 8b21c128-50e4-4fba-896c-e83a60c86382 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-79732-predictor-77bcc96986-82s7n namespace: kserve-ci-e2e-test resourceVersion: "16932" uid: e3ffb519-60f9-4431-baf6-36792845e4b2 kind: Event lastTimestamp: "2026-04-16T18:47:20Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 949ms (949ms including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-16T18:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:20Z" name: message-dumper-raw-79732-predictor-77bcc96986-82s7n.18a6eac63869f803 namespace: kserve-ci-e2e-test resourceVersion: "16969" uid: 0667168e-0762-4aae-88cb-7ff1b83c94ef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-79732-predictor-77bcc96986-82s7n namespace: kserve-ci-e2e-test resourceVersion: "16932" uid: e3ffb519-60f9-4431-baf6-36792845e4b2 kind: Event lastTimestamp: "2026-04-16T18:47:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:20Z" name: message-dumper-raw-79732-predictor-77bcc96986-82s7n.18a6eac63b4c3219 namespace: kserve-ci-e2e-test resourceVersion: "16970" uid: cd8becbe-4a7b-4982-8490-14565bd9cfe6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-79732-predictor-77bcc96986-82s7n namespace: kserve-ci-e2e-test resourceVersion: "16932" uid: e3ffb519-60f9-4431-baf6-36792845e4b2 kind: Event lastTimestamp: "2026-04-16T18:47:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:20Z" name: message-dumper-raw-79732-predictor-77bcc96986-82s7n.18a6eac63bb9fbba namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: bab54afc-772c-4c82-8c9d-1f1d5335d4b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-79732-predictor-77bcc96986-82s7n namespace: kserve-ci-e2e-test resourceVersion: "16932" uid: e3ffb519-60f9-4431-baf6-36792845e4b2 kind: Event lastTimestamp: "2026-04-16T18:48:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:48:54Z" name: message-dumper-raw-79732-predictor-77bcc96986-82s7n.18a6eadc3c158f6d namespace: kserve-ci-e2e-test resourceVersion: "17806" uid: 2b96b2ed-e2e5-41ff-96f1-edcced10affa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-79732-predictor-77bcc96986 namespace: kserve-ci-e2e-test resourceVersion: "16918" uid: bdb2533e-a193-4981-b6ca-7442a16dd970 kind: Event lastTimestamp: "2026-04-16T18:47:19Z" message: 'Created pod: message-dumper-raw-79732-predictor-77bcc96986-82s7n' metadata: creationTimestamp: "2026-04-16T18:47:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:47:19Z" name: message-dumper-raw-79732-predictor-77bcc96986.18a6eac5f714a943 namespace: kserve-ci-e2e-test resourceVersion: "16933" uid: 0eec37aa-e6f0-4b1d-ae24-4e2b83b31bb2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-79732-predictor namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: d19f1451-1650-4a53-9148-c330392fec69 kind: Event lastTimestamp: "2026-04-16T18:47:18Z" message: Scaled up replica set message-dumper-raw-79732-predictor-77bcc96986 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:47:18Z" name: message-dumper-raw-79732-predictor.18a6eac5f3df42db namespace: kserve-ci-e2e-test resourceVersion: "16921" uid: c1c1058e-cf6f-4993-a80d-78e62626452d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:47:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-79732-predictor namespace: kserve-ci-e2e-test resourceVersion: "16917" uid: 89d8c639-35b3-42cc-90d8-206c5dba10ef kind: Event lastTimestamp: "2026-04-16T18:47:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:47:49Z" name: message-dumper-raw-79732-predictor.18a6eac972ec2323 namespace: kserve-ci-e2e-test resourceVersion: "17258" uid: b49a2ea5-2a3e-443d-b9d2-c1b51a393c21 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:47:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-79732-predictor namespace: kserve-ci-e2e-test resourceVersion: "16917" uid: 89d8c639-35b3-42cc-90d8-206c5dba10ef kind: Event lastTimestamp: "2026-04-16T18:47:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:47:49Z" name: message-dumper-raw-79732-predictor.18a6eac973532c7c namespace: kserve-ci-e2e-test resourceVersion: "17259" uid: 16bd1db4-c8c5-44d2-900a-2ffc62d71ba0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:47:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-79732 namespace: kserve-ci-e2e-test resourceVersion: "16897" uid: 85abbbbc-2400-4ee8-ab31-f6c96e360911 kind: Event lastTimestamp: "2026-04-16T18:47:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:47:20Z" name: message-dumper-raw-79732.18a6eac5f3b5568c namespace: kserve-ci-e2e-test resourceVersion: "16986" uid: 6cc01774-4283-4d51-a943-8599353c1bef reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-79732 namespace: kserve-ci-e2e-test resourceVersion: "16897" uid: 85abbbbc-2400-4ee8-ab31-f6c96e360911 kind: Event lastTimestamp: "2026-04-16T18:47:18Z" message: 'Failed to update status for InferenceService "message-dumper-raw-79732": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-79732": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:47:18Z" name: message-dumper-raw-79732.18a6eac5f463e759 namespace: kserve-ci-e2e-test resourceVersion: "16925" uid: 64c45067-b0e1-42e5-9b06-3294291400c1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-79732 namespace: kserve-ci-e2e-test resourceVersion: "16983" uid: 85abbbbc-2400-4ee8-ab31-f6c96e360911 kind: Event lastTimestamp: "2026-04-16T18:47:20Z" message: InferenceService [message-dumper-raw-79732] is Ready metadata: creationTimestamp: "2026-04-16T18:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:47:20Z" name: message-dumper-raw-79732.18a6eac647fdde0d namespace: kserve-ci-e2e-test resourceVersion: "16985" uid: 7855b2ad-c57b-455b-a19a-5849c9bb1d31 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:45:28.439496Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-1a6d3-78c446778d-kdhcl namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: c5b114e0-d6bb-432d-889e-02ebc68c44a8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-1a6d3-78c446778d-kdhcl to ip-10-0-142-225.ec2.internal metadata: creationTimestamp: "2026-04-16T18:45:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:45:28Z" name: model-chainer-raw-1a6d3-78c446778d-kdhcl.18a6eaac38071cd7 namespace: kserve-ci-e2e-test resourceVersion: "15841" uid: 7bf5fa50-1a52-4f69-a07f-cba587027f66 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9d7c89bb-dzjdk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:28Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-1a6d3-78c446778d-kdhcl namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: c5b114e0-d6bb-432d-889e-02ebc68c44a8 kind: Event lastTimestamp: "2026-04-16T18:45:28Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:45:28Z" name: model-chainer-raw-1a6d3-78c446778d-kdhcl.18a6eaac50c74a14 namespace: kserve-ci-e2e-test resourceVersion: "15857" uid: 03b1c60f-68bf-4f8d-99f8-6a5d538d750f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-1a6d3} kind: Pod name: model-chainer-raw-1a6d3-78c446778d-kdhcl namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: c5b114e0-d6bb-432d-889e-02ebc68c44a8 kind: Event lastTimestamp: "2026-04-16T18:45:28Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:743cff1ceb833e46d1b8e65ba261d1d549e2e90054da6d533b54a9a2f13462c7" metadata: creationTimestamp: "2026-04-16T18:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:28Z" name: model-chainer-raw-1a6d3-78c446778d-kdhcl.18a6eaac51fa2627 namespace: kserve-ci-e2e-test resourceVersion: "15859" uid: 81f2ca59-c4f3-4d4d-8d4a-76bd5ae0a0a3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-1a6d3} kind: Pod name: model-chainer-raw-1a6d3-78c446778d-kdhcl namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: c5b114e0-d6bb-432d-889e-02ebc68c44a8 kind: Event lastTimestamp: "2026-04-16T18:45:31Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:743cff1ceb833e46d1b8e65ba261d1d549e2e90054da6d533b54a9a2f13462c7" in 2.456s (2.456s including waiting). Image size: 216230482 bytes.' metadata: creationTimestamp: "2026-04-16T18:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:31Z" name: model-chainer-raw-1a6d3-78c446778d-kdhcl.18a6eaace467343f namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: 12aeb4f3-100b-4817-919e-ac441df804fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-1a6d3} kind: Pod name: model-chainer-raw-1a6d3-78c446778d-kdhcl namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: c5b114e0-d6bb-432d-889e-02ebc68c44a8 kind: Event lastTimestamp: "2026-04-16T18:45:32Z" message: 'Created container: model-chainer-raw-1a6d3' metadata: creationTimestamp: "2026-04-16T18:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:32Z" name: model-chainer-raw-1a6d3-78c446778d-kdhcl.18a6eaace71dcccc namespace: kserve-ci-e2e-test resourceVersion: "15885" uid: 0e363f8a-4ceb-4e5b-b1b6-ad22e41d1a2d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-1a6d3} kind: Pod name: model-chainer-raw-1a6d3-78c446778d-kdhcl namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: c5b114e0-d6bb-432d-889e-02ebc68c44a8 kind: Event lastTimestamp: "2026-04-16T18:45:32Z" message: Started container model-chainer-raw-1a6d3 metadata: creationTimestamp: "2026-04-16T18:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:32Z" name: model-chainer-raw-1a6d3-78c446778d-kdhcl.18a6eaace78d3dd9 namespace: kserve-ci-e2e-test resourceVersion: "15886" uid: 56ead1e4-a600-4e25-9b81-e885a84843bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-1a6d3} kind: Pod name: model-chainer-raw-1a6d3-78c446778d-kdhcl namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: c5b114e0-d6bb-432d-889e-02ebc68c44a8 kind: Event lastTimestamp: "2026-04-16T18:45:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:743cff1ceb833e46d1b8e65ba261d1d549e2e90054da6d533b54a9a2f13462c7" already present on machine metadata: creationTimestamp: "2026-04-16T18:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:31Z" name: model-chainer-raw-1a6d3-78c446778d-kdhcl.18a6eaad0bb1c2dc namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: 3ba6e111-de6e-4d80-8aca-e664bff05dd8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-1a6d3} kind: Pod name: model-chainer-raw-1a6d3-78c446778d-kdhcl namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: c5b114e0-d6bb-432d-889e-02ebc68c44a8 kind: Event lastTimestamp: "2026-04-16T18:45:34Z" message: Back-off restarting failed container model-chainer-raw-1a6d3 in pod model-chainer-raw-1a6d3-78c446778d-kdhcl_kserve-ci-e2e-test(c5b114e0-d6bb-432d-889e-02ebc68c44a8) metadata: creationTimestamp: "2026-04-16T18:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:45:35Z" name: model-chainer-raw-1a6d3-78c446778d-kdhcl.18a6eaad47740f67 namespace: kserve-ci-e2e-test resourceVersion: "15909" uid: 0a666c06-b2b5-4a31-81f4-5cc4bc6a5ca7 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-1a6d3-78c446778d namespace: kserve-ci-e2e-test resourceVersion: "15829" uid: d1f3580b-0d5e-423a-99a3-76098c89dbc2 kind: Event lastTimestamp: "2026-04-16T18:45:28Z" message: 'Created pod: model-chainer-raw-1a6d3-78c446778d-kdhcl' metadata: creationTimestamp: "2026-04-16T18:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:45:28Z" name: model-chainer-raw-1a6d3-78c446778d.18a6eaac37966d3e namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: f05d9df8-3215-4000-95e7-410d78190fab reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-1a6d3 namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: 81783650-717f-4125-bf9d-301a238697e0 kind: Event lastTimestamp: "2026-04-16T18:45:28Z" message: Scaled up replica set model-chainer-raw-1a6d3-78c446778d from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:45:28Z" name: model-chainer-raw-1a6d3.18a6eaac366745d9 namespace: kserve-ci-e2e-test resourceVersion: "15831" uid: c798cda0-205e-4a14-9752-d311d25569c6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:45:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-1a6d3 namespace: kserve-ci-e2e-test resourceVersion: "15850" uid: ac32aef4-7fa9-4595-8c34-4e002a3018cd kind: Event lastTimestamp: "2026-04-16T18:45:28Z" message: InferenceGraph [model-chainer-raw-1a6d3] is Ready metadata: creationTimestamp: "2026-04-16T18:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:45:28Z" name: model-chainer-raw-1a6d3.18a6eaac3cddb0ef namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: 214ea247-c58c-4ef4-b4d3-0c85b207dda3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:47:08.758976Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-fd0ec-79c8d5fc87-88r29 namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: f376ce5e-0bdb-4112-9699-d3138855f6ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-fd0ec-79c8d5fc87-88r29 to ip-10-0-142-225.ec2.internal metadata: creationTimestamp: "2026-04-16T18:47:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:47:08Z" name: model-chainer-raw-hpa-fd0ec-79c8d5fc87-88r29.18a6eac39388e61b namespace: kserve-ci-e2e-test resourceVersion: "16776" uid: 9ce14473-aebd-434f-ad20-68bd024ae93e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9d7c89bb-dzjdk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:09Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-fd0ec-79c8d5fc87-88r29 namespace: kserve-ci-e2e-test resourceVersion: "16780" uid: f376ce5e-0bdb-4112-9699-d3138855f6ba kind: Event lastTimestamp: "2026-04-16T18:47:09Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:47:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:47:09Z" name: model-chainer-raw-hpa-fd0ec-79c8d5fc87-88r29.18a6eac3ac352fab namespace: kserve-ci-e2e-test resourceVersion: "16793" uid: ffb04c16-62c6-42aa-94c4-7d0583ec41bb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:47:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-fd0ec} kind: Pod name: model-chainer-raw-hpa-fd0ec-79c8d5fc87-88r29 namespace: kserve-ci-e2e-test resourceVersion: "16774" uid: f376ce5e-0bdb-4112-9699-d3138855f6ba kind: Event lastTimestamp: "2026-04-16T18:47:10Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:743cff1ceb833e46d1b8e65ba261d1d549e2e90054da6d533b54a9a2f13462c7" already present on machine metadata: creationTimestamp: "2026-04-16T18:47:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:10Z" name: model-chainer-raw-hpa-fd0ec-79c8d5fc87-88r29.18a6eac3ad6cdc55 namespace: kserve-ci-e2e-test resourceVersion: "16805" uid: ff4a30e5-b8bc-4877-87eb-4a3593a55087 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:47:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-fd0ec} kind: Pod name: model-chainer-raw-hpa-fd0ec-79c8d5fc87-88r29 namespace: kserve-ci-e2e-test resourceVersion: "16774" uid: f376ce5e-0bdb-4112-9699-d3138855f6ba kind: Event lastTimestamp: "2026-04-16T18:47:10Z" message: 'Created container: model-chainer-raw-hpa-fd0ec' metadata: creationTimestamp: "2026-04-16T18:47:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:10Z" name: model-chainer-raw-hpa-fd0ec-79c8d5fc87-88r29.18a6eac3afed32b1 namespace: kserve-ci-e2e-test resourceVersion: "16808" uid: 31f034ba-f248-4608-b2ee-10721a5a07f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:47:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-fd0ec} kind: Pod name: model-chainer-raw-hpa-fd0ec-79c8d5fc87-88r29 namespace: kserve-ci-e2e-test resourceVersion: "16774" uid: f376ce5e-0bdb-4112-9699-d3138855f6ba kind: Event lastTimestamp: "2026-04-16T18:47:10Z" message: Started container model-chainer-raw-hpa-fd0ec metadata: creationTimestamp: "2026-04-16T18:47:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:10Z" name: model-chainer-raw-hpa-fd0ec-79c8d5fc87-88r29.18a6eac3b059d4f9 namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: 2982200b-05fe-4e43-a8a3-5a535e707895 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:47:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-fd0ec} kind: Pod name: model-chainer-raw-hpa-fd0ec-79c8d5fc87-88r29 namespace: kserve-ci-e2e-test resourceVersion: "16774" uid: f376ce5e-0bdb-4112-9699-d3138855f6ba kind: Event lastTimestamp: "2026-04-16T18:47:14Z" message: Back-off restarting failed container model-chainer-raw-hpa-fd0ec in pod model-chainer-raw-hpa-fd0ec-79c8d5fc87-88r29_kserve-ci-e2e-test(f376ce5e-0bdb-4112-9699-d3138855f6ba) metadata: creationTimestamp: "2026-04-16T18:47:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:14Z" name: model-chainer-raw-hpa-fd0ec-79c8d5fc87-88r29.18a6eac428c6971b namespace: kserve-ci-e2e-test resourceVersion: "16846" uid: 89734013-c37b-482e-8e05-9ec2f77f42be reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-142-225.ec2.internal source: component: kubelet host: ip-10-0-142-225.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-fd0ec-79c8d5fc87 namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 2ef00f2e-0437-4c96-a633-0a20fec5cc91 kind: Event lastTimestamp: "2026-04-16T18:47:08Z" message: 'Created pod: model-chainer-raw-hpa-fd0ec-79c8d5fc87-88r29' metadata: creationTimestamp: "2026-04-16T18:47:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:47:08Z" name: model-chainer-raw-hpa-fd0ec-79c8d5fc87.18a6eac392e7357f namespace: kserve-ci-e2e-test resourceVersion: "16772" uid: 4dccd3fc-9ca4-4633-8d42-eb91d8b56443 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-fd0ec namespace: kserve-ci-e2e-test resourceVersion: "16762" uid: d101c4b5-6d52-46dd-a18b-354061ccdbad kind: Event lastTimestamp: "2026-04-16T18:47:08Z" message: Scaled up replica set model-chainer-raw-hpa-fd0ec-79c8d5fc87 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:47:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:47:08Z" name: model-chainer-raw-hpa-fd0ec.18a6eac39193875d namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: 08ebf55f-8e76-451e-bfac-fb4e328c3dd7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:08Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-fd0ec namespace: kserve-ci-e2e-test resourceVersion: "16785" uid: a34cae4c-9518-48d7-bb4c-0da1bfe2ed14 kind: Event lastTimestamp: "2026-04-16T18:47:08Z" message: InferenceGraph [model-chainer-raw-hpa-fd0ec] is Ready metadata: creationTimestamp: "2026-04-16T18:47:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:47:08Z" name: model-chainer-raw-hpa-fd0ec.18a6eac3975e5194 namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: a2b4622a-b606-4a1e-99c2-d7f93309d473 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:53:06.123600Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-dd577-predictor-7758bff698-bxx9x namespace: kserve-ci-e2e-test resourceVersion: "19965" uid: 7dfe33f4-89ad-46f9-9a1b-77218902c940 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-dd577-predictor-7758bff698-bxx9x to ip-10-0-140-1.ec2.internal metadata: creationTimestamp: "2026-04-16T18:53:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:53:06Z" name: raw-sklearn-dd577-predictor-7758bff698-bxx9x.18a6eb16c8204ed1 namespace: kserve-ci-e2e-test resourceVersion: "19970" uid: 608b4c5a-6b17-4826-9a82-0d178bfa4ea5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9d7c89bb-dzjdk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:53:06Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-dd577-predictor-7758bff698-bxx9x namespace: kserve-ci-e2e-test resourceVersion: "19977" uid: 7dfe33f4-89ad-46f9-9a1b-77218902c940 kind: Event lastTimestamp: "2026-04-16T18:53:06Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:53:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:53:06Z" name: raw-sklearn-dd577-predictor-7758bff698-bxx9x.18a6eb16e15670da namespace: kserve-ci-e2e-test resourceVersion: "19990" uid: 72280511-cb11-41db-9074-4ed36ad1b2c4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:53:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-dd577-predictor-7758bff698-bxx9x namespace: kserve-ci-e2e-test resourceVersion: "19968" uid: 7dfe33f4-89ad-46f9-9a1b-77218902c940 kind: Event lastTimestamp: "2026-04-16T18:53:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T18:53:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:53:06Z" name: raw-sklearn-dd577-predictor-7758bff698-bxx9x.18a6eb16e2a7b32c namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 56c0c4a3-2ae8-459b-9b45-9e91df107a9e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:53:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-dd577-predictor-7758bff698-bxx9x namespace: kserve-ci-e2e-test resourceVersion: "19968" uid: 7dfe33f4-89ad-46f9-9a1b-77218902c940 kind: Event lastTimestamp: "2026-04-16T18:53:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:53:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:53:06Z" name: raw-sklearn-dd577-predictor-7758bff698-bxx9x.18a6eb16e5c435a6 namespace: kserve-ci-e2e-test resourceVersion: "19994" uid: ffca8fa4-b545-416e-b970-d018d8a8a08f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:53:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-dd577-predictor-7758bff698-bxx9x namespace: kserve-ci-e2e-test resourceVersion: "19968" uid: 7dfe33f4-89ad-46f9-9a1b-77218902c940 kind: Event lastTimestamp: "2026-04-16T18:53:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:53:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:53:06Z" name: raw-sklearn-dd577-predictor-7758bff698-bxx9x.18a6eb16e6379a0c namespace: kserve-ci-e2e-test resourceVersion: "19995" uid: f4480078-0d0f-4d28-a011-5892a37a4d62 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-dd577-predictor-7758bff698-bxx9x namespace: kserve-ci-e2e-test resourceVersion: "19968" uid: 7dfe33f4-89ad-46f9-9a1b-77218902c940 kind: Event lastTimestamp: "2026-04-16T18:53:11Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1326" already present on machine metadata: creationTimestamp: "2026-04-16T18:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:53:11Z" name: raw-sklearn-dd577-predictor-7758bff698-bxx9x.18a6eb17f8e720b0 namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: ae0ad4af-e3d8-44d3-89c1-1ca8a9deaa12 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-dd577-predictor-7758bff698-bxx9x namespace: kserve-ci-e2e-test resourceVersion: "19968" uid: 7dfe33f4-89ad-46f9-9a1b-77218902c940 kind: Event lastTimestamp: "2026-04-16T18:53:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:53:11Z" name: raw-sklearn-dd577-predictor-7758bff698-bxx9x.18a6eb17fcb8bf73 namespace: kserve-ci-e2e-test resourceVersion: "20039" uid: 244f6483-e6cb-465a-84dc-2897928449d7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-dd577-predictor-7758bff698-bxx9x namespace: kserve-ci-e2e-test resourceVersion: "19968" uid: 7dfe33f4-89ad-46f9-9a1b-77218902c940 kind: Event lastTimestamp: "2026-04-16T18:53:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:53:11Z" name: raw-sklearn-dd577-predictor-7758bff698-bxx9x.18a6eb17fd42f8af namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: a7040b32-a766-4f0e-9e3e-7836b99e6d5f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T18:53:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-dd577-predictor-7758bff698-bxx9x namespace: kserve-ci-e2e-test resourceVersion: "19968" uid: 7dfe33f4-89ad-46f9-9a1b-77218902c940 kind: Event lastTimestamp: "2026-04-16T18:54:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:53:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:54:13Z" name: raw-sklearn-dd577-predictor-7758bff698-bxx9x.18a6eb1834f18a22 namespace: kserve-ci-e2e-test resourceVersion: "20484" uid: 7ffa25ad-460f-4550-b96b-c514cd33ff73 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:54:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-dd577-predictor-7758bff698-bxx9x namespace: kserve-ci-e2e-test resourceVersion: "19968" uid: 7dfe33f4-89ad-46f9-9a1b-77218902c940 kind: Event lastTimestamp: "2026-04-16T18:54:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:54:26Z" name: raw-sklearn-dd577-predictor-7758bff698-bxx9x.18a6eb296edff91b namespace: kserve-ci-e2e-test resourceVersion: "20607" uid: d88194d4-ac5e-40df-9ced-627f27cde475 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:53:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-dd577-predictor-7758bff698 namespace: kserve-ci-e2e-test resourceVersion: "19956" uid: 7e545735-2d6d-4f80-b868-c934c30f6d7e kind: Event lastTimestamp: "2026-04-16T18:53:06Z" message: 'Created pod: raw-sklearn-dd577-predictor-7758bff698-bxx9x' metadata: creationTimestamp: "2026-04-16T18:53:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:53:06Z" name: raw-sklearn-dd577-predictor-7758bff698.18a6eb16c7b2e68a namespace: kserve-ci-e2e-test resourceVersion: "19967" uid: e9309002-b375-4f63-9cc4-784c25de88f2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:53:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-dd577-predictor namespace: kserve-ci-e2e-test resourceVersion: "19954" uid: 5c44153e-42b1-432c-9989-05d7cabfa7f1 kind: Event lastTimestamp: "2026-04-16T18:53:06Z" message: Scaled up replica set raw-sklearn-dd577-predictor-7758bff698 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:53:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:53:06Z" name: raw-sklearn-dd577-predictor.18a6eb16c544eab2 namespace: kserve-ci-e2e-test resourceVersion: "19958" uid: 3c5baef3-bafd-48b2-8823-e0d82bc36cd7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:53:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-dd577-predictor namespace: kserve-ci-e2e-test resourceVersion: "19957" uid: 500edf85-ae5a-4fd2-8d18-7ff4bfc08b26 kind: Event lastTimestamp: "2026-04-16T18:53:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:53:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:53:36Z" name: raw-sklearn-dd577-predictor.18a6eb1a43ee2e1a namespace: kserve-ci-e2e-test resourceVersion: "20220" uid: e0cdc2f8-65cf-4946-b912-8d311b4f59d3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:53:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-dd577-predictor namespace: kserve-ci-e2e-test resourceVersion: "19957" uid: 500edf85-ae5a-4fd2-8d18-7ff4bfc08b26 kind: Event lastTimestamp: "2026-04-16T18:53:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:53:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:53:36Z" name: raw-sklearn-dd577-predictor.18a6eb1a4447b2e5 namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: 12679d87-03ff-44a4-93e2-dc58842f7b73 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:53:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-dd577-predictor namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: 500edf85-ae5a-4fd2-8d18-7ff4bfc08b26 kind: Event lastTimestamp: "2026-04-16T18:54:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:54:21Z" name: raw-sklearn-dd577-predictor.18a6eb21425f8c10 namespace: kserve-ci-e2e-test resourceVersion: "20540" uid: 5c095e40-27f7-4ec0-93f3-480da21b9700 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:53:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-dd577-predictor namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: 500edf85-ae5a-4fd2-8d18-7ff4bfc08b26 kind: Event lastTimestamp: "2026-04-16T18:54:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:54:21Z" name: raw-sklearn-dd577-predictor.18a6eb2142c1c96f namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: fe607bd0-eea0-4b81-8cf9-4173ef1046be reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T18:53:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-dd577 namespace: kserve-ci-e2e-test resourceVersion: "19944" uid: 13d3c073-9d00-4c34-942e-1a283788699b kind: Event lastTimestamp: "2026-04-16T18:54:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:53:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:54:23Z" name: raw-sklearn-dd577.18a6eb16c548afc0 namespace: kserve-ci-e2e-test resourceVersion: "20561" uid: 5dbfb18f-0501-45b0-86e0-139aa4770ba9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:53:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-dd577 namespace: kserve-ci-e2e-test resourceVersion: "19944" uid: 13d3c073-9d00-4c34-942e-1a283788699b kind: Event lastTimestamp: "2026-04-16T18:53:06Z" message: 'Failed to update status for InferenceService "raw-sklearn-dd577": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-dd577": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:53:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:53:06Z" name: raw-sklearn-dd577.18a6eb16c5f8dd19 namespace: kserve-ci-e2e-test resourceVersion: "19986" uid: e702c7dd-5978-40c7-ac65-91f3edb1e5c6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:53:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-dd577 namespace: kserve-ci-e2e-test resourceVersion: "19963" uid: 13d3c073-9d00-4c34-942e-1a283788699b kind: Event lastTimestamp: "2026-04-16T18:53:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-dd577": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:53:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:53:06Z" name: raw-sklearn-dd577.18a6eb16cca6b3a0 namespace: kserve-ci-e2e-test resourceVersion: "19988" uid: ba6faee1-eb39-4270-91bd-cf78d23640db reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:54:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-dd577 namespace: kserve-ci-e2e-test resourceVersion: "20559" uid: 13d3c073-9d00-4c34-942e-1a283788699b kind: Event lastTimestamp: "2026-04-16T18:54:23Z" message: InferenceService [raw-sklearn-dd577] is Ready metadata: creationTimestamp: "2026-04-16T18:54:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:54:23Z" name: raw-sklearn-dd577.18a6eb28c165be61 namespace: kserve-ci-e2e-test resourceVersion: "20560" uid: 50b71592-e64d-4098-8e9f-0185cd15d0cc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:54:26.348873Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8 namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: b4a471bc-ffc4-457c-a68f-fc06acb635be kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8 to ip-10-0-140-1.ec2.internal metadata: creationTimestamp: "2026-04-16T18:54:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:54:26Z" name: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8.18a6eb2975ecc7d8 namespace: kserve-ci-e2e-test resourceVersion: "20632" uid: d0a15069-b9ba-4e5f-b1cc-f8d6d26a94f4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9d7c89bb-dzjdk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:54:26Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8 namespace: kserve-ci-e2e-test resourceVersion: "20634" uid: b4a471bc-ffc4-457c-a68f-fc06acb635be kind: Event lastTimestamp: "2026-04-16T18:54:26Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:54:26Z" name: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8.18a6eb298ec4d463 namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: f0d9b521-6880-4439-a17c-1d951df04a65 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:54:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8 namespace: kserve-ci-e2e-test resourceVersion: "20626" uid: b4a471bc-ffc4-457c-a68f-fc06acb635be kind: Event lastTimestamp: "2026-04-16T18:54:26Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:54:26Z" name: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8.18a6eb299009c09a namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: 865accf8-7f4b-42c2-b8b3-3a0cf503e4e6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:54:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8 namespace: kserve-ci-e2e-test resourceVersion: "20626" uid: b4a471bc-ffc4-457c-a68f-fc06acb635be kind: Event lastTimestamp: "2026-04-16T18:54:26Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:54:26Z" name: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8.18a6eb2992f084ec namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: 0cda6541-67a5-474e-b991-0e8ad17c800e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:54:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8 namespace: kserve-ci-e2e-test resourceVersion: "20626" uid: b4a471bc-ffc4-457c-a68f-fc06acb635be kind: Event lastTimestamp: "2026-04-16T18:54:26Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:54:26Z" name: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8.18a6eb2993625458 namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: 3a7cbe58-12d8-4126-9147-c34dea390700 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:54:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8 namespace: kserve-ci-e2e-test resourceVersion: "20626" uid: b4a471bc-ffc4-457c-a68f-fc06acb635be kind: Event lastTimestamp: "2026-04-16T18:54:31Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1326" already present on machine metadata: creationTimestamp: "2026-04-16T18:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:54:31Z" name: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8.18a6eb2aaab26975 namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: d08a06ff-7d82-4fa9-b0e7-fb50d8adb147 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:54:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8 namespace: kserve-ci-e2e-test resourceVersion: "20626" uid: b4a471bc-ffc4-457c-a68f-fc06acb635be kind: Event lastTimestamp: "2026-04-16T18:54:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:54:31Z" name: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8.18a6eb2aae4ce13b namespace: kserve-ci-e2e-test resourceVersion: "20691" uid: 9428dc19-87d7-495d-b74a-2e5871cccdee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:54:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8 namespace: kserve-ci-e2e-test resourceVersion: "20626" uid: b4a471bc-ffc4-457c-a68f-fc06acb635be kind: Event lastTimestamp: "2026-04-16T18:54:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:54:31Z" name: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8.18a6eb2aaec0346d namespace: kserve-ci-e2e-test resourceVersion: "20692" uid: 64fd2d1e-ed25-44b5-a0e6-dc42085776a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T18:54:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8 namespace: kserve-ci-e2e-test resourceVersion: "20626" uid: b4a471bc-ffc4-457c-a68f-fc06acb635be kind: Event lastTimestamp: "2026-04-16T18:55:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:55:33Z" name: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8.18a6eb2ae6b90863 namespace: kserve-ci-e2e-test resourceVersion: "21133" uid: 94691d8b-3e7b-4858-b2d9-9025910179c6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:55:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8 namespace: kserve-ci-e2e-test resourceVersion: "20626" uid: b4a471bc-ffc4-457c-a68f-fc06acb635be kind: Event lastTimestamp: "2026-04-16T18:55:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:55:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:55:46Z" name: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8.18a6eb3c1ac087ad namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: b67c5b49-f34f-484e-a34d-930f40fe1ad4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-1.ec2.internal source: component: kubelet host: ip-10-0-140-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:54:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-cef54-predictor-9df4fddf namespace: kserve-ci-e2e-test resourceVersion: "20613" uid: bd549981-f87d-496a-b130-a1579579341c kind: Event lastTimestamp: "2026-04-16T18:54:26Z" message: 'Created pod: raw-sklearn-runtime-cef54-predictor-9df4fddf-kgpx8' metadata: creationTimestamp: "2026-04-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:54:26Z" name: raw-sklearn-runtime-cef54-predictor-9df4fddf.18a6eb29752a42b4 namespace: kserve-ci-e2e-test resourceVersion: "20627" uid: 1a05f7ad-84dd-466a-a04e-89304c1a1856 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:54:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-cef54-predictor namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: 04f3e39d-f5c9-4f64-baaa-3b76b38f45e9 kind: Event lastTimestamp: "2026-04-16T18:54:26Z" message: Scaled up replica set raw-sklearn-runtime-cef54-predictor-9df4fddf from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:54:26Z" name: raw-sklearn-runtime-cef54-predictor.18a6eb297020d5b6 namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: 5162260e-21eb-4c4f-9805-69cb38aa00a5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:54:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-cef54-predictor namespace: kserve-ci-e2e-test resourceVersion: "20612" uid: 22d798cf-2bd7-4111-bd93-c4b4ed73fdf1 kind: Event lastTimestamp: "2026-04-16T18:54:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:54:41Z" name: raw-sklearn-runtime-cef54-predictor.18a6eb2cefaff1b8 namespace: kserve-ci-e2e-test resourceVersion: "20764" uid: 65ffa751-45ef-432f-b9da-a7c2063985b0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:54:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-cef54-predictor namespace: kserve-ci-e2e-test resourceVersion: "20612" uid: 22d798cf-2bd7-4111-bd93-c4b4ed73fdf1 kind: Event lastTimestamp: "2026-04-16T18:54:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:54:41Z" name: raw-sklearn-runtime-cef54-predictor.18a6eb2cf0795170 namespace: kserve-ci-e2e-test resourceVersion: "20765" uid: 4104b04d-e6a1-48c1-a224-20342a690b41 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:54:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-cef54-predictor namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: 22d798cf-2bd7-4111-bd93-c4b4ed73fdf1 kind: Event lastTimestamp: "2026-04-16T18:55:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:55:41Z" name: raw-sklearn-runtime-cef54-predictor.18a6eb306f4c9436 namespace: kserve-ci-e2e-test resourceVersion: "21185" uid: 7ec77e50-7674-40cd-b33e-7a068910f7b9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:54:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-cef54-predictor namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: 22d798cf-2bd7-4111-bd93-c4b4ed73fdf1 kind: Event lastTimestamp: "2026-04-16T18:55:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:55:41Z" name: raw-sklearn-runtime-cef54-predictor.18a6eb306fdb5326 namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: 3a106ba0-f9c2-4b75-a528-b70c1f1b0822 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T18:54:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-cef54 namespace: kserve-ci-e2e-test resourceVersion: "20587" uid: febc255a-1a06-4ca0-83be-83e0d3d0396b kind: Event lastTimestamp: "2026-04-16T18:55:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:55:43Z" name: raw-sklearn-runtime-cef54.18a6eb296ffb6d92 namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 487be8e5-7b63-4fc3-8f12-9ac6a69c549b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:54:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-cef54 namespace: kserve-ci-e2e-test resourceVersion: "20587" uid: febc255a-1a06-4ca0-83be-83e0d3d0396b kind: Event lastTimestamp: "2026-04-16T18:54:26Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-cef54": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-cef54": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:54:26Z" name: raw-sklearn-runtime-cef54.18a6eb297054bc91 namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: 203086e9-d982-4555-9030-937db22899ec reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:55:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-cef54 namespace: kserve-ci-e2e-test resourceVersion: "21209" uid: febc255a-1a06-4ca0-83be-83e0d3d0396b kind: Event lastTimestamp: "2026-04-16T18:55:43Z" message: InferenceService [raw-sklearn-runtime-cef54] is Ready metadata: creationTimestamp: "2026-04-16T18:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:55:43Z" name: raw-sklearn-runtime-cef54.18a6eb3b72fdb71f namespace: kserve-ci-e2e-test resourceVersion: "21210" uid: a3f87dd8-7f17-471e-9576-58a501526697 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "21428"