--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-24T14:45:24.309633Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-3988b2-predictor-6ccd758dd9-gdp2k namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: a3da90f3-7d28-4390-8bac-c36d5ff927c3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-3988b2-predictor-6ccd758dd9-gdp2k to ip-10-0-143-92.ec2.internal metadata: creationTimestamp: "2026-04-24T14:45:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T14:45:24Z" name: isvc-init-fail-3988b2-predictor-6ccd758dd9-gdp2k.18a952370973ed84 namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: d74d6eea-41ce-4535-92e3-27bd59385cca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd58b56dd-ls688 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:24Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-3988b2-predictor-6ccd758dd9-gdp2k namespace: kserve-ci-e2e-test resourceVersion: "19697" uid: a3da90f3-7d28-4390-8bac-c36d5ff927c3 kind: Event lastTimestamp: "2026-04-24T14:45:24Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T14:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T14:45:24Z" name: isvc-init-fail-3988b2-predictor-6ccd758dd9-gdp2k.18a9523722afa9d0 namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: ee2300cd-86c7-42ff-9497-12363db75ef1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T14:45:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-3988b2-predictor-6ccd758dd9-gdp2k namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: a3da90f3-7d28-4390-8bac-c36d5ff927c3 kind: Event lastTimestamp: "2026-04-24T14:45:30Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3370fbcd366f44a8710ac5ab5a02712bec0322005baa64fee46cbff50aae7c18" already present on machine metadata: creationTimestamp: "2026-04-24T14:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:45:30Z" name: isvc-init-fail-3988b2-predictor-6ccd758dd9-gdp2k.18a9523724036192 namespace: kserve-ci-e2e-test resourceVersion: "19771" uid: 724b291f-61cc-4672-8884-528bb3d8adee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T14:45:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-3988b2-predictor-6ccd758dd9-gdp2k namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: a3da90f3-7d28-4390-8bac-c36d5ff927c3 kind: Event lastTimestamp: "2026-04-24T14:45:30Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T14:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:45:30Z" name: isvc-init-fail-3988b2-predictor-6ccd758dd9-gdp2k.18a9523727524215 namespace: kserve-ci-e2e-test resourceVersion: "19775" uid: c23a2415-06b2-49d9-92b6-f815d477f3c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T14:45:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-3988b2-predictor-6ccd758dd9-gdp2k namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: a3da90f3-7d28-4390-8bac-c36d5ff927c3 kind: Event lastTimestamp: "2026-04-24T14:45:30Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T14:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:45:31Z" name: isvc-init-fail-3988b2-predictor-6ccd758dd9-gdp2k.18a9523727c72f76 namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: 2b35ff56-b924-42a8-be68-b54300057897 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-3988b2-predictor-6ccd758dd9-gdp2k namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: a3da90f3-7d28-4390-8bac-c36d5ff927c3 kind: Event lastTimestamp: "2026-04-24T14:45:34Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-04-24T14:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:45:34Z" name: isvc-init-fail-3988b2-predictor-6ccd758dd9-gdp2k.18a9523960122495 namespace: kserve-ci-e2e-test resourceVersion: "19818" uid: 7490b67b-b107-4930-b39d-587157215514 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-3988b2-predictor-6ccd758dd9 namespace: kserve-ci-e2e-test resourceVersion: "19680" uid: 78de2c5e-685d-4b56-ab85-098c1b3c72d4 kind: Event lastTimestamp: "2026-04-24T14:45:24Z" message: 'Created pod: isvc-init-fail-3988b2-predictor-6ccd758dd9-gdp2k' metadata: creationTimestamp: "2026-04-24T14:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:45:24Z" name: isvc-init-fail-3988b2-predictor-6ccd758dd9.18a952370912215d namespace: kserve-ci-e2e-test resourceVersion: "19691" uid: 18c64d78-a1ff-4268-8d24-3807594cabf1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-3988b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: 45b0e89e-7eb0-4296-b743-39131ef7ffe3 kind: Event lastTimestamp: "2026-04-24T14:45:24Z" message: Scaled up replica set isvc-init-fail-3988b2-predictor-6ccd758dd9 from 0 to 1 metadata: creationTimestamp: "2026-04-24T14:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:45:24Z" name: isvc-init-fail-3988b2-predictor.18a9523705e30241 namespace: kserve-ci-e2e-test resourceVersion: "19682" uid: 752e1c0f-5017-40a9-9ed9-91fe56fc3449 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-24T14:45:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-3988b2 namespace: kserve-ci-e2e-test resourceVersion: "19654" uid: fa771d97-6ea0-47d9-bf19-e73f9827c31a kind: Event lastTimestamp: "2026-04-24T14:45:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T14:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:45:31Z" name: isvc-init-fail-3988b2.18a9523705bcc783 namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 91397e0f-af6e-4e14-8f49-1576651b1762 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T14:45:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-3988b2 namespace: kserve-ci-e2e-test resourceVersion: "19654" uid: fa771d97-6ea0-47d9-bf19-e73f9827c31a kind: Event lastTimestamp: "2026-04-24T14:45:24Z" message: 'Failed to update status for InferenceService "isvc-init-fail-3988b2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-3988b2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T14:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:45:24Z" name: isvc-init-fail-3988b2.18a95237062955ca namespace: kserve-ci-e2e-test resourceVersion: "19699" uid: 37255dd6-7d0c-4af6-9fb2-9f86286fb590 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T14:45:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-3988b2 namespace: kserve-ci-e2e-test resourceVersion: "19654" uid: fa771d97-6ea0-47d9-bf19-e73f9827c31a kind: Event lastTimestamp: "2026-04-24T14:45:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-3988b2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T14:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:45:24Z" name: isvc-init-fail-3988b2.18a952370629b3f9 namespace: kserve-ci-e2e-test resourceVersion: "19701" uid: ec3b2ae1-13b9-4697-9905-2a4bac37d95d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T14:39:54.436393Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx namespace: kserve-ci-e2e-test resourceVersion: "16970" uid: 620113a7-5642-4e53-8146-88ebb8525670 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-cf657-predictor-5749968549-bdvcx to ip-10-0-143-92.ec2.internal metadata: creationTimestamp: "2026-04-24T14:39:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T14:39:54Z" name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx.18a951ea3b79aa3a namespace: kserve-ci-e2e-test resourceVersion: "16974" uid: 5f14826c-5347-4be4-8394-85c50964c4c3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd58b56dd-ls688 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:54Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx namespace: kserve-ci-e2e-test resourceVersion: "16978" uid: 620113a7-5642-4e53-8146-88ebb8525670 kind: Event lastTimestamp: "2026-04-24T14:39:54Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T14:39:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T14:39:54Z" name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx.18a951ea544bfc36 namespace: kserve-ci-e2e-test resourceVersion: "16985" uid: 6d14a7c7-3ad7-49b1-885d-cccd5331e62b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: 620113a7-5642-4e53-8146-88ebb8525670 kind: Event lastTimestamp: "2026-04-24T14:39:54Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3370fbcd366f44a8710ac5ab5a02712bec0322005baa64fee46cbff50aae7c18" already present on machine metadata: creationTimestamp: "2026-04-24T14:39:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:54Z" name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx.18a951ea55ce617a namespace: kserve-ci-e2e-test resourceVersion: "16988" uid: 45f072fa-c8ff-4edd-9d47-7d230948fa23 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: 620113a7-5642-4e53-8146-88ebb8525670 kind: Event lastTimestamp: "2026-04-24T14:39:54Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T14:39:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:54Z" name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx.18a951ea58c72287 namespace: kserve-ci-e2e-test resourceVersion: "16992" uid: 3959810d-4604-4210-9a52-cd14735c9230 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: 620113a7-5642-4e53-8146-88ebb8525670 kind: Event lastTimestamp: "2026-04-24T14:39:54Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T14:39:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:54Z" name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx.18a951ea5939d594 namespace: kserve-ci-e2e-test resourceVersion: "16993" uid: b7ff9fa1-17b5-489d-860f-161da5dd959f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: 620113a7-5642-4e53-8146-88ebb8525670 kind: Event lastTimestamp: "2026-04-24T14:39:58Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-04-24T14:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:58Z" name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx.18a951eb48930f21 namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 2576cb00-67d3-4593-83cf-55faf2e2d5c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: 620113a7-5642-4e53-8146-88ebb8525670 kind: Event lastTimestamp: "2026-04-24T14:39:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T14:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:59Z" name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx.18a951eb4bdafc73 namespace: kserve-ci-e2e-test resourceVersion: "17029" uid: c8c379ad-c734-42be-915a-e09d3c65db38 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: 620113a7-5642-4e53-8146-88ebb8525670 kind: Event lastTimestamp: "2026-04-24T14:39:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T14:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:59Z" name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx.18a951eb4c519933 namespace: kserve-ci-e2e-test resourceVersion: "17030" uid: 983a67d8-9311-4d6c-9e6c-9e2bdc85f293 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: 620113a7-5642-4e53-8146-88ebb8525670 kind: Event lastTimestamp: "2026-04-24T14:39:59Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:a5c29c96f4a50b100eb461f2aad96676c97347e2c3d4bdbb37c6548004d27dcb" already present on machine metadata: creationTimestamp: "2026-04-24T14:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:59Z" name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx.18a951eb4c5d9ad2 namespace: kserve-ci-e2e-test resourceVersion: "17031" uid: 4328f9b5-1241-4442-8466-8a30397c2a61 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: 620113a7-5642-4e53-8146-88ebb8525670 kind: Event lastTimestamp: "2026-04-24T14:39:59Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-24T14:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:59Z" name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx.18a951eb4f13241e namespace: kserve-ci-e2e-test resourceVersion: "17033" uid: d37dfe06-126d-4a97-ab4a-e34f3672e390 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: 620113a7-5642-4e53-8146-88ebb8525670 kind: Event lastTimestamp: "2026-04-24T14:39:59Z" message: Started container agent metadata: creationTimestamp: "2026-04-24T14:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:59Z" name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx.18a951eb4f84279d namespace: kserve-ci-e2e-test resourceVersion: "17034" uid: 7580c513-545d-4f6f-bf6c-7bb9e8cd2130 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-24T14:39:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: 620113a7-5642-4e53-8146-88ebb8525670 kind: Event lastTimestamp: "2026-04-24T14:41:40Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T14:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:41:40Z" name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx.18a951eb84875472 namespace: kserve-ci-e2e-test resourceVersion: "17885" uid: f06ded49-05a5-44da-81dc-f3ba4910bf5f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-24T14:39:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: 620113a7-5642-4e53-8146-88ebb8525670 kind: Event lastTimestamp: "2026-04-24T14:41:40Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T14:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:41:40Z" name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx.18a951eb84918a53 namespace: kserve-ci-e2e-test resourceVersion: "17886" uid: 3e675374-31b0-420b-b19a-6f6e8171d16d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:41:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: 620113a7-5642-4e53-8146-88ebb8525670 kind: Event lastTimestamp: "2026-04-24T14:41:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T14:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:41:19Z" name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx.18a951fe12ab1908 namespace: kserve-ci-e2e-test resourceVersion: "17669" uid: 10904a02-b30f-43ce-9621-701c8cfe3b5e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:41:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: 620113a7-5642-4e53-8146-88ebb8525670 kind: Event lastTimestamp: "2026-04-24T14:41:19Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-24T14:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:41:19Z" name: isvc-logger-raw-cf657-predictor-5749968549-bdvcx.18a951fe12ac63f2 namespace: kserve-ci-e2e-test resourceVersion: "17670" uid: cd1edf4e-74ca-406f-8ccb-b8570f9a2872 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-cf657-predictor-5749968549 namespace: kserve-ci-e2e-test resourceVersion: "16959" uid: 48bee503-6938-4c9e-869f-ec41f44ea153 kind: Event lastTimestamp: "2026-04-24T14:39:54Z" message: 'Created pod: isvc-logger-raw-cf657-predictor-5749968549-bdvcx' metadata: creationTimestamp: "2026-04-24T14:39:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:39:54Z" name: isvc-logger-raw-cf657-predictor-5749968549.18a951ea3afc916e namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: 3a7ba4b6-2208-41b8-be6c-8d934dab1ae2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-cf657-predictor namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: c3f2d5f5-0549-4ec4-8719-55c9604d503f kind: Event lastTimestamp: "2026-04-24T14:39:54Z" message: Scaled up replica set isvc-logger-raw-cf657-predictor-5749968549 from 0 to 1 metadata: creationTimestamp: "2026-04-24T14:39:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:39:54Z" name: isvc-logger-raw-cf657-predictor.18a951ea37333c2c namespace: kserve-ci-e2e-test resourceVersion: "16961" uid: 21913f2f-7c84-469b-aed6-6ba3cd973559 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T14:40:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-cf657-predictor namespace: kserve-ci-e2e-test resourceVersion: "16958" uid: 1e82a07d-67be-4cb4-a68d-584e83ec8dbb kind: Event lastTimestamp: "2026-04-24T14:40:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:40:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:40:39Z" name: isvc-logger-raw-cf657-predictor.18a951edb658a45f namespace: kserve-ci-e2e-test resourceVersion: "17344" uid: 1562bfb3-619f-4e16-8e17-74bc9f236407 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T14:40:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-cf657-predictor namespace: kserve-ci-e2e-test resourceVersion: "16958" uid: 1e82a07d-67be-4cb4-a68d-584e83ec8dbb kind: Event lastTimestamp: "2026-04-24T14:40:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:40:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:40:39Z" name: isvc-logger-raw-cf657-predictor.18a951edb6c19a18 namespace: kserve-ci-e2e-test resourceVersion: "17345" uid: a6727403-b1ec-4233-baa5-a78fb616a001 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T14:40:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-cf657-predictor namespace: kserve-ci-e2e-test resourceVersion: "17121" uid: 1e82a07d-67be-4cb4-a68d-584e83ec8dbb kind: Event lastTimestamp: "2026-04-24T14:41:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T14:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:41:09Z" name: isvc-logger-raw-cf657-predictor.18a951f1357ccb4e namespace: kserve-ci-e2e-test resourceVersion: "17557" uid: 1c8530a8-273e-4669-ab12-88a15fb72729 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T14:40:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-cf657-predictor namespace: kserve-ci-e2e-test resourceVersion: "17121" uid: 1e82a07d-67be-4cb4-a68d-584e83ec8dbb kind: Event lastTimestamp: "2026-04-24T14:41:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T14:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:41:09Z" name: isvc-logger-raw-cf657-predictor.18a951f135e99726 namespace: kserve-ci-e2e-test resourceVersion: "17558" uid: 7ae611ac-c20e-4fb5-be60-53d5cd154c37 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T14:39:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-cf657 namespace: kserve-ci-e2e-test resourceVersion: "16947" uid: 975ada6c-8f56-444a-8834-825dea30f307 kind: Event lastTimestamp: "2026-04-24T14:41:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T14:39:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:41:11Z" name: isvc-logger-raw-cf657.18a951ea370d128b namespace: kserve-ci-e2e-test resourceVersion: "17579" uid: 4a40b9d4-b80c-4810-b431-23aa687ba9e5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-cf657 namespace: kserve-ci-e2e-test resourceVersion: "16947" uid: 975ada6c-8f56-444a-8834-825dea30f307 kind: Event lastTimestamp: "2026-04-24T14:39:54Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-cf657": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-cf657": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T14:39:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:39:54Z" name: isvc-logger-raw-cf657.18a951ea37b776cf namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: 77fe0f7d-c064-485e-98a4-9eec90d59d08 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:41:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-cf657 namespace: kserve-ci-e2e-test resourceVersion: "17577" uid: 975ada6c-8f56-444a-8834-825dea30f307 kind: Event lastTimestamp: "2026-04-24T14:41:11Z" message: InferenceService [isvc-logger-raw-cf657] is Ready metadata: creationTimestamp: "2026-04-24T14:41:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:41:11Z" name: isvc-logger-raw-cf657.18a951fc11e5ecb4 namespace: kserve-ci-e2e-test resourceVersion: "17578" uid: b8b6da3b-424d-4fd2-84cb-1422e5ce3948 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T14:43:49.948482Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65 namespace: kserve-ci-e2e-test resourceVersion: "18847" uid: 3ed1b3fd-1a1c-4ab8-a44d-bd6efbf52a29 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65 to ip-10-0-143-92.ec2.internal metadata: creationTimestamp: "2026-04-24T14:43:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T14:43:49Z" name: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65.18a952211116fb05 namespace: kserve-ci-e2e-test resourceVersion: "18852" uid: f599effa-536b-435d-a948-9123a813f033 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd58b56dd-ls688 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:43:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65 namespace: kserve-ci-e2e-test resourceVersion: "18855" uid: 3ed1b3fd-1a1c-4ab8-a44d-bd6efbf52a29 kind: Event lastTimestamp: "2026-04-24T14:43:50Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T14:43:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T14:43:50Z" name: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65.18a952212a9b68b4 namespace: kserve-ci-e2e-test resourceVersion: "18866" uid: cecbf65c-f442-481b-ae46-07478b103a5b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:43:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65 namespace: kserve-ci-e2e-test resourceVersion: "18850" uid: 3ed1b3fd-1a1c-4ab8-a44d-bd6efbf52a29 kind: Event lastTimestamp: "2026-04-24T14:43:50Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3370fbcd366f44a8710ac5ab5a02712bec0322005baa64fee46cbff50aae7c18" already present on machine metadata: creationTimestamp: "2026-04-24T14:43:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:43:50Z" name: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65.18a952212be56039 namespace: kserve-ci-e2e-test resourceVersion: "18868" uid: 78fce152-2c72-4fd1-84c3-6d67546ea3cc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:43:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65 namespace: kserve-ci-e2e-test resourceVersion: "18850" uid: 3ed1b3fd-1a1c-4ab8-a44d-bd6efbf52a29 kind: Event lastTimestamp: "2026-04-24T14:43:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T14:43:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:43:50Z" name: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65.18a952212f00cc6a namespace: kserve-ci-e2e-test resourceVersion: "18869" uid: 892c781b-43ca-43bb-8f53-d8d457651434 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:43:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65 namespace: kserve-ci-e2e-test resourceVersion: "18850" uid: 3ed1b3fd-1a1c-4ab8-a44d-bd6efbf52a29 kind: Event lastTimestamp: "2026-04-24T14:43:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T14:43:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:43:50Z" name: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65.18a952212f72c669 namespace: kserve-ci-e2e-test resourceVersion: "18870" uid: effc40d2-5e61-4ff2-ac0d-ed9bbc1a3fee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65 namespace: kserve-ci-e2e-test resourceVersion: "18850" uid: 3ed1b3fd-1a1c-4ab8-a44d-bd6efbf52a29 kind: Event lastTimestamp: "2026-04-24T14:43:54Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-04-24T14:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:43:54Z" name: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65.18a9522228da4ca2 namespace: kserve-ci-e2e-test resourceVersion: "18904" uid: 9c86e969-7401-4b6f-8f1a-959af6742dc5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65 namespace: kserve-ci-e2e-test resourceVersion: "18850" uid: 3ed1b3fd-1a1c-4ab8-a44d-bd6efbf52a29 kind: Event lastTimestamp: "2026-04-24T14:43:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T14:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:43:54Z" name: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65.18a952222c0d67ef namespace: kserve-ci-e2e-test resourceVersion: "18908" uid: 3e0957d4-92a3-4f47-a0f5-c1e978c7c527 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65 namespace: kserve-ci-e2e-test resourceVersion: "18850" uid: 3ed1b3fd-1a1c-4ab8-a44d-bd6efbf52a29 kind: Event lastTimestamp: "2026-04-24T14:43:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T14:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:43:54Z" name: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65.18a952222c825eeb namespace: kserve-ci-e2e-test resourceVersion: "18909" uid: c1df5374-bd2e-4233-a264-414c6f4fd9c2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T14:43:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65 namespace: kserve-ci-e2e-test resourceVersion: "18850" uid: 3ed1b3fd-1a1c-4ab8-a44d-bd6efbf52a29 kind: Event lastTimestamp: "2026-04-24T14:45:26Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T14:43:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:45:26Z" name: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65.18a9522264c649b1 namespace: kserve-ci-e2e-test resourceVersion: "19738" uid: 1ab96e41-f38a-44e2-b7d1-89153bc43ac0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65 namespace: kserve-ci-e2e-test resourceVersion: "18850" uid: 3ed1b3fd-1a1c-4ab8-a44d-bd6efbf52a29 kind: Event lastTimestamp: "2026-04-24T14:45:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T14:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:45:24Z" name: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65.18a9523704ae9774 namespace: kserve-ci-e2e-test resourceVersion: "19674" uid: 7e5c00d3-2a11-4bfb-821e-59e93c0c82de reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:43:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-c6676b-predictor-5dc44dd8b8 namespace: kserve-ci-e2e-test resourceVersion: "18838" uid: e63448e7-60f3-4ee6-a958-e750f9fab42e kind: Event lastTimestamp: "2026-04-24T14:43:49Z" message: 'Created pod: isvc-primary-c6676b-predictor-5dc44dd8b8-7jh65' metadata: creationTimestamp: "2026-04-24T14:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:43:49Z" name: isvc-primary-c6676b-predictor-5dc44dd8b8.18a9522110ab0ca1 namespace: kserve-ci-e2e-test resourceVersion: "18849" uid: 6fd36409-8dbf-4053-bb40-ed5ae09c6a48 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:43:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-c6676b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18836" uid: 5e19b689-9dd6-4f8c-b3d6-126df4a59a37 kind: Event lastTimestamp: "2026-04-24T14:43:49Z" message: Scaled up replica set isvc-primary-c6676b-predictor-5dc44dd8b8 from 0 to 1 metadata: creationTimestamp: "2026-04-24T14:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:43:49Z" name: isvc-primary-c6676b-predictor.18a952210c7221a2 namespace: kserve-ci-e2e-test resourceVersion: "18840" uid: 1e8c7c5f-62a3-4317-b6b0-a049e858e7d4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:44:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-c6676b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18837" uid: c5ea5b1a-bcfa-4ff3-ad4b-c12ab891231f kind: Event lastTimestamp: "2026-04-24T14:44:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:44:04Z" name: isvc-primary-c6676b-predictor.18a952248b53308c namespace: kserve-ci-e2e-test resourceVersion: "18991" uid: e9dec9b2-d382-4aa8-9d82-e8909c52853f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:44:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-c6676b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18837" uid: c5ea5b1a-bcfa-4ff3-ad4b-c12ab891231f kind: Event lastTimestamp: "2026-04-24T14:44:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:44:04Z" name: isvc-primary-c6676b-predictor.18a952248bb6b0b3 namespace: kserve-ci-e2e-test resourceVersion: "18993" uid: 61899914-d175-440d-b8f0-0f29bce55690 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T14:44:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-c6676b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18992" uid: c5ea5b1a-bcfa-4ff3-ad4b-c12ab891231f kind: Event lastTimestamp: "2026-04-24T14:45:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T14:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:45:19Z" name: isvc-primary-c6676b-predictor.18a952280a7871ec namespace: kserve-ci-e2e-test resourceVersion: "19599" uid: 89deff22-24b6-4ed6-a8fc-61c1098c3edc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T14:44:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-c6676b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18992" uid: c5ea5b1a-bcfa-4ff3-ad4b-c12ab891231f kind: Event lastTimestamp: "2026-04-24T14:45:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T14:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:45:19Z" name: isvc-primary-c6676b-predictor.18a952280ac094fb namespace: kserve-ci-e2e-test resourceVersion: "19600" uid: 7ca23ea7-1147-43c8-8f65-53a8c032772c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-04-24T14:43:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-c6676b namespace: kserve-ci-e2e-test resourceVersion: "18811" uid: 5bb7e6c6-9afe-4fbb-9ef9-1946d5e4f077 kind: Event lastTimestamp: "2026-04-24T14:45:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T14:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:45:06Z" name: isvc-primary-c6676b.18a952210c4cd6dd namespace: kserve-ci-e2e-test resourceVersion: "19438" uid: b66847c3-432a-46d8-a73d-b6a6c3eeb521 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:43:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-c6676b namespace: kserve-ci-e2e-test resourceVersion: "18811" uid: 5bb7e6c6-9afe-4fbb-9ef9-1946d5e4f077 kind: Event lastTimestamp: "2026-04-24T14:43:49Z" message: 'Failed to update status for InferenceService "isvc-primary-c6676b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-c6676b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T14:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:43:49Z" name: isvc-primary-c6676b.18a952210cbcb303 namespace: kserve-ci-e2e-test resourceVersion: "18842" uid: 28350155-23ef-43dd-ad3e-e9f06f01959c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:43:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-c6676b namespace: kserve-ci-e2e-test resourceVersion: "18811" uid: 5bb7e6c6-9afe-4fbb-9ef9-1946d5e4f077 kind: Event lastTimestamp: "2026-04-24T14:43:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-c6676b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T14:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:43:49Z" name: isvc-primary-c6676b.18a952210cbcfdcc namespace: kserve-ci-e2e-test resourceVersion: "18843" uid: 13858eec-e93b-4c2b-ae58-bfc519e98385 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-c6676b namespace: kserve-ci-e2e-test resourceVersion: "19436" uid: 5bb7e6c6-9afe-4fbb-9ef9-1946d5e4f077 kind: Event lastTimestamp: "2026-04-24T14:45:06Z" message: InferenceService [isvc-primary-c6676b] is Ready metadata: creationTimestamp: "2026-04-24T14:45:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:45:06Z" name: isvc-primary-c6676b.18a95232f1598f33 namespace: kserve-ci-e2e-test resourceVersion: "19437" uid: 6d669c35-ede9-4bbd-9bcf-5e108a6e35a1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T14:34:38.474114Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7 namespace: kserve-ci-e2e-test resourceVersion: "14070" uid: 8573857b-afac-4a29-90a2-a368527c028b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7 to ip-10-0-143-92.ec2.internal metadata: creationTimestamp: "2026-04-24T14:34:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T14:34:38Z" name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7.18a951a0aaa7ff38 namespace: kserve-ci-e2e-test resourceVersion: "14077" uid: 5d115131-e245-4133-8630-c0bb27b6b3dc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd58b56dd-ls688 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:34:38Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7 namespace: kserve-ci-e2e-test resourceVersion: "14082" uid: 8573857b-afac-4a29-90a2-a368527c028b kind: Event lastTimestamp: "2026-04-24T14:34:38Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T14:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T14:34:38Z" name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7.18a951a0c3837936 namespace: kserve-ci-e2e-test resourceVersion: "14092" uid: b3911d2d-9fa6-4267-876c-6d0b06734149 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:34:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7 namespace: kserve-ci-e2e-test resourceVersion: "14073" uid: 8573857b-afac-4a29-90a2-a368527c028b kind: Event lastTimestamp: "2026-04-24T14:34:38Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:3370fbcd366f44a8710ac5ab5a02712bec0322005baa64fee46cbff50aae7c18" metadata: creationTimestamp: "2026-04-24T14:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:34:38Z" name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7.18a951a0c4e3ea7f namespace: kserve-ci-e2e-test resourceVersion: "14094" uid: ebedf24b-13c7-438a-88e5-71252e884f46 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:34:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7 namespace: kserve-ci-e2e-test resourceVersion: "14073" uid: 8573857b-afac-4a29-90a2-a368527c028b kind: Event lastTimestamp: "2026-04-24T14:34:42Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:3370fbcd366f44a8710ac5ab5a02712bec0322005baa64fee46cbff50aae7c18" in 3.567s (3.567s including waiting). Image size: 300957076 bytes.' metadata: creationTimestamp: "2026-04-24T14:34:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:34:42Z" name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7.18a951a1998c9b08 namespace: kserve-ci-e2e-test resourceVersion: "14119" uid: 5ac4890c-32a2-49d1-ae90-87157e71bc25 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:34:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7 namespace: kserve-ci-e2e-test resourceVersion: "14073" uid: 8573857b-afac-4a29-90a2-a368527c028b kind: Event lastTimestamp: "2026-04-24T14:34:42Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T14:34:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:34:42Z" name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7.18a951a19cb2b883 namespace: kserve-ci-e2e-test resourceVersion: "14120" uid: 7751d85e-da17-488a-ae35-ea1773f09a57 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:34:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7 namespace: kserve-ci-e2e-test resourceVersion: "14073" uid: 8573857b-afac-4a29-90a2-a368527c028b kind: Event lastTimestamp: "2026-04-24T14:34:42Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T14:34:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:34:42Z" name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7.18a951a19d24c0c6 namespace: kserve-ci-e2e-test resourceVersion: "14121" uid: 7b682b01-5252-41ef-aacf-91fa32eebeb1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:34:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7 namespace: kserve-ci-e2e-test resourceVersion: "14073" uid: 8573857b-afac-4a29-90a2-a368527c028b kind: Event lastTimestamp: "2026-04-24T14:34:46Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" metadata: creationTimestamp: "2026-04-24T14:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:34:46Z" name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7.18a951a2a5abf001 namespace: kserve-ci-e2e-test resourceVersion: "14157" uid: 5d392689-b9bf-4db4-9cb0-7fc2d9179bb0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:35:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7 namespace: kserve-ci-e2e-test resourceVersion: "14073" uid: 8573857b-afac-4a29-90a2-a368527c028b kind: Event lastTimestamp: "2026-04-24T14:35:00Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" in 13.087s (13.087s including waiting). Image size: 1560705952 bytes.' metadata: creationTimestamp: "2026-04-24T14:35:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:35:00Z" name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7.18a951a5b1bd9957 namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: 96e1acee-035e-42de-9ec6-e1361bc3d3bc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:35:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7 namespace: kserve-ci-e2e-test resourceVersion: "14073" uid: 8573857b-afac-4a29-90a2-a368527c028b kind: Event lastTimestamp: "2026-04-24T14:35:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T14:35:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:35:00Z" name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7.18a951a5b4e9ce44 namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 08f1498b-81da-4a63-bfcb-a1463802eb00 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:35:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7 namespace: kserve-ci-e2e-test resourceVersion: "14073" uid: 8573857b-afac-4a29-90a2-a368527c028b kind: Event lastTimestamp: "2026-04-24T14:35:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T14:35:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:35:00Z" name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7.18a951a5b55ed0bc namespace: kserve-ci-e2e-test resourceVersion: "14258" uid: 6352db56-c601-4ae2-ae8c-ff048baa800a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:35:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7 namespace: kserve-ci-e2e-test resourceVersion: "14073" uid: 8573857b-afac-4a29-90a2-a368527c028b kind: Event lastTimestamp: "2026-04-24T14:35:00Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:a5c29c96f4a50b100eb461f2aad96676c97347e2c3d4bdbb37c6548004d27dcb" metadata: creationTimestamp: "2026-04-24T14:35:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:35:00Z" name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7.18a951a5b56bb2c8 namespace: kserve-ci-e2e-test resourceVersion: "14259" uid: 6c0da9cd-bdd9-487c-848e-0212e9b6aaad reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:35:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7 namespace: kserve-ci-e2e-test resourceVersion: "14073" uid: 8573857b-afac-4a29-90a2-a368527c028b kind: Event lastTimestamp: "2026-04-24T14:35:02Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:a5c29c96f4a50b100eb461f2aad96676c97347e2c3d4bdbb37c6548004d27dcb" in 2.764s (2.764s including waiting). Image size: 237896243 bytes.' metadata: creationTimestamp: "2026-04-24T14:35:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:35:02Z" name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7.18a951a65a3743b2 namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 9fbe123f-c9da-4565-93a7-c217339361bb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:35:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7 namespace: kserve-ci-e2e-test resourceVersion: "14073" uid: 8573857b-afac-4a29-90a2-a368527c028b kind: Event lastTimestamp: "2026-04-24T14:35:02Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-24T14:35:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:35:02Z" name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7.18a951a65d0c0f65 namespace: kserve-ci-e2e-test resourceVersion: "14280" uid: 3a122f0e-8967-406b-adb3-2abfe5283b66 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:35:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7 namespace: kserve-ci-e2e-test resourceVersion: "14073" uid: 8573857b-afac-4a29-90a2-a368527c028b kind: Event lastTimestamp: "2026-04-24T14:35:02Z" message: Started container agent metadata: creationTimestamp: "2026-04-24T14:35:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:35:02Z" name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7.18a951a65d83fe08 namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 406f8b36-705e-4d6d-a4cd-26c60efab7b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T14:35:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7 namespace: kserve-ci-e2e-test resourceVersion: "14073" uid: 8573857b-afac-4a29-90a2-a368527c028b kind: Event lastTimestamp: "2026-04-24T14:36:48Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T14:35:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:36:48Z" name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7.18a951a662cfa4ee namespace: kserve-ci-e2e-test resourceVersion: "15181" uid: 4c77f12b-2710-4899-9e5c-c6f74b7939ef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-24T14:35:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7 namespace: kserve-ci-e2e-test resourceVersion: "14073" uid: 8573857b-afac-4a29-90a2-a368527c028b kind: Event lastTimestamp: "2026-04-24T14:36:48Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T14:35:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:36:48Z" name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7.18a951a69e8911ca namespace: kserve-ci-e2e-test resourceVersion: "15182" uid: f0be1596-ee33-4b28-94ca-b022c022f868 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7 namespace: kserve-ci-e2e-test resourceVersion: "14073" uid: 8573857b-afac-4a29-90a2-a368527c028b kind: Event lastTimestamp: "2026-04-24T14:36:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T14:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:36:23Z" name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7.18a951b927f159fd namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: efd779a5-e33b-4cec-b929-dd4edc8b9d38 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7 namespace: kserve-ci-e2e-test resourceVersion: "14073" uid: 8573857b-afac-4a29-90a2-a368527c028b kind: Event lastTimestamp: "2026-04-24T14:36:23Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-24T14:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:36:23Z" name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7.18a951b927f293ab namespace: kserve-ci-e2e-test resourceVersion: "14898" uid: 7043f185-99f7-4c43-984f-fc2376165f97 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:34:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5 namespace: kserve-ci-e2e-test resourceVersion: "14062" uid: de529250-58c8-4bbc-b04d-0060b1ede6da kind: Event lastTimestamp: "2026-04-24T14:34:38Z" message: 'Created pod: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5-c2xf7' metadata: creationTimestamp: "2026-04-24T14:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:34:38Z" name: isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5.18a951a0aa38fdf3 namespace: kserve-ci-e2e-test resourceVersion: "14072" uid: 9f97e733-1dd3-4e18-8ddf-c55904778fe9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:34:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-26556-predictor namespace: kserve-ci-e2e-test resourceVersion: "14061" uid: e45474dc-1833-459d-8f88-baa859c22fe3 kind: Event lastTimestamp: "2026-04-24T14:34:38Z" message: Scaled up replica set isvc-raw-sklearn-batcher-26556-predictor-54fd76b6d5 from 0 to 1 metadata: creationTimestamp: "2026-04-24T14:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:34:38Z" name: isvc-raw-sklearn-batcher-26556-predictor.18a951a0a3cf2772 namespace: kserve-ci-e2e-test resourceVersion: "14063" uid: 65b0415d-ddce-45c4-b1b3-9a8bf29a646d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T14:34:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-26556-predictor namespace: kserve-ci-e2e-test resourceVersion: "14069" uid: 100362db-a6fc-4cfe-b290-b138a330ab48 kind: Event lastTimestamp: "2026-04-24T14:35:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:34:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:35:38Z" name: isvc-raw-sklearn-batcher-26556-predictor.18a951a42952d62e namespace: kserve-ci-e2e-test resourceVersion: "14536" uid: 560cf4c9-9b25-4e15-8cba-3de899196f8f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T14:34:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-26556-predictor namespace: kserve-ci-e2e-test resourceVersion: "14069" uid: 100362db-a6fc-4cfe-b290-b138a330ab48 kind: Event lastTimestamp: "2026-04-24T14:35:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:34:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:35:38Z" name: isvc-raw-sklearn-batcher-26556-predictor.18a951a429f328a4 namespace: kserve-ci-e2e-test resourceVersion: "14537" uid: 57260601-aabb-4dff-a195-c12c8669d311 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T14:35:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-26556-predictor namespace: kserve-ci-e2e-test resourceVersion: "14206" uid: 100362db-a6fc-4cfe-b290-b138a330ab48 kind: Event lastTimestamp: "2026-04-24T14:36:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T14:35:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:36:23Z" name: isvc-raw-sklearn-batcher-26556-predictor.18a951b225ca7aca namespace: kserve-ci-e2e-test resourceVersion: "14871" uid: 9b8947a9-2947-4eba-980b-0c260177f951 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T14:35:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-26556-predictor namespace: kserve-ci-e2e-test resourceVersion: "14206" uid: 100362db-a6fc-4cfe-b290-b138a330ab48 kind: Event lastTimestamp: "2026-04-24T14:36:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T14:35:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:36:23Z" name: isvc-raw-sklearn-batcher-26556-predictor.18a951b226385a0a namespace: kserve-ci-e2e-test resourceVersion: "14872" uid: 42a41b88-7516-44fa-b92b-c457fcedf923 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T14:34:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-26556 namespace: kserve-ci-e2e-test resourceVersion: "14049" uid: 352bf26f-fa86-44a3-9fc5-610b0fa48745 kind: Event lastTimestamp: "2026-04-24T14:36:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T14:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:36:08Z" name: isvc-raw-sklearn-batcher-26556.18a951a0a9eb1418 namespace: kserve-ci-e2e-test resourceVersion: "14758" uid: f0b30464-c8c9-4706-8c00-709433507cd6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:34:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-26556 namespace: kserve-ci-e2e-test resourceVersion: "14049" uid: 352bf26f-fa86-44a3-9fc5-610b0fa48745 kind: Event lastTimestamp: "2026-04-24T14:34:38Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-26556": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-26556": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T14:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:34:38Z" name: isvc-raw-sklearn-batcher-26556.18a951a0aa4aa2af namespace: kserve-ci-e2e-test resourceVersion: "14075" uid: e4cf05fb-f767-46b4-9da0-0f70d46d88d8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:34:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-26556 namespace: kserve-ci-e2e-test resourceVersion: "14049" uid: 352bf26f-fa86-44a3-9fc5-610b0fa48745 kind: Event lastTimestamp: "2026-04-24T14:34:38Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-26556": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T14:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:34:38Z" name: isvc-raw-sklearn-batcher-26556.18a951a0aa4afb7b namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: a25e4c70-72c1-4a46-8019-44054ee1dcd9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-26556 namespace: kserve-ci-e2e-test resourceVersion: "14755" uid: 352bf26f-fa86-44a3-9fc5-610b0fa48745 kind: Event lastTimestamp: "2026-04-24T14:36:08Z" message: InferenceService [isvc-raw-sklearn-batcher-26556] is Ready metadata: creationTimestamp: "2026-04-24T14:36:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:36:08Z" name: isvc-raw-sklearn-batcher-26556.18a951b593ebc6f8 namespace: kserve-ci-e2e-test resourceVersion: "14757" uid: a929819b-fe4b-4269-9adf-92a4434936bb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T14:45:10.087682Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-c6676b-predictor-6df7fcb885-6g8bn namespace: kserve-ci-e2e-test resourceVersion: "19488" uid: 8e3fecf5-c573-4500-8493-843d4230c9b1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-c6676b-predictor-6df7fcb885-6g8bn to ip-10-0-143-92.ec2.internal metadata: creationTimestamp: "2026-04-24T14:45:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T14:45:10Z" name: isvc-secondary-c6676b-predictor-6df7fcb885-6g8bn.18a95233b9c21c63 namespace: kserve-ci-e2e-test resourceVersion: "19493" uid: cf43afc4-0f1d-4d51-bb8d-f26e72632259 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd58b56dd-ls688 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-c6676b-predictor-6df7fcb885-6g8bn namespace: kserve-ci-e2e-test resourceVersion: "19496" uid: 8e3fecf5-c573-4500-8493-843d4230c9b1 kind: Event lastTimestamp: "2026-04-24T14:45:10Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T14:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T14:45:10Z" name: isvc-secondary-c6676b-predictor-6df7fcb885-6g8bn.18a95233d28d3283 namespace: kserve-ci-e2e-test resourceVersion: "19507" uid: 9999dbc8-17b6-4960-b596-fb458505eb16 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T14:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-c6676b-predictor-6df7fcb885-6g8bn namespace: kserve-ci-e2e-test resourceVersion: "19490" uid: 8e3fecf5-c573-4500-8493-843d4230c9b1 kind: Event lastTimestamp: "2026-04-24T14:45:12Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3370fbcd366f44a8710ac5ab5a02712bec0322005baa64fee46cbff50aae7c18" already present on machine metadata: creationTimestamp: "2026-04-24T14:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:45:12Z" name: isvc-secondary-c6676b-predictor-6df7fcb885-6g8bn.18a95233d3df929e namespace: kserve-ci-e2e-test resourceVersion: "19535" uid: df267fbd-2a72-457b-8cdd-4b6bd96ddae3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T14:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-c6676b-predictor-6df7fcb885-6g8bn namespace: kserve-ci-e2e-test resourceVersion: "19490" uid: 8e3fecf5-c573-4500-8493-843d4230c9b1 kind: Event lastTimestamp: "2026-04-24T14:45:12Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T14:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:45:12Z" name: isvc-secondary-c6676b-predictor-6df7fcb885-6g8bn.18a95233d6f30e5f namespace: kserve-ci-e2e-test resourceVersion: "19539" uid: b7464ad0-3996-4b30-9bf5-d4d7616b3aca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T14:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-c6676b-predictor-6df7fcb885-6g8bn namespace: kserve-ci-e2e-test resourceVersion: "19490" uid: 8e3fecf5-c573-4500-8493-843d4230c9b1 kind: Event lastTimestamp: "2026-04-24T14:45:12Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T14:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:45:12Z" name: isvc-secondary-c6676b-predictor-6df7fcb885-6g8bn.18a95233d77a510b namespace: kserve-ci-e2e-test resourceVersion: "19540" uid: 5aa1074c-fcdc-4c46-9430-c33e49404fea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-c6676b-predictor-6df7fcb885-6g8bn namespace: kserve-ci-e2e-test resourceVersion: "19490" uid: 8e3fecf5-c573-4500-8493-843d4230c9b1 kind: Event lastTimestamp: "2026-04-24T14:45:16Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-c6676b-predictor-6df7fcb885-6g8bn_kserve-ci-e2e-test(8e3fecf5-c573-4500-8493-843d4230c9b1) metadata: creationTimestamp: "2026-04-24T14:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:45:16Z" name: isvc-secondary-c6676b-predictor-6df7fcb885-6g8bn.18a952354ff7bace namespace: kserve-ci-e2e-test resourceVersion: "19572" uid: d9857e8d-bc31-4357-9661-1525cf821e8c reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-c6676b-predictor-6df7fcb885 namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: 385620dd-a85e-432b-bd9f-6c4fb03a0021 kind: Event lastTimestamp: "2026-04-24T14:45:10Z" message: 'Created pod: isvc-secondary-c6676b-predictor-6df7fcb885-6g8bn' metadata: creationTimestamp: "2026-04-24T14:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:45:10Z" name: isvc-secondary-c6676b-predictor-6df7fcb885.18a95233b9527f0e namespace: kserve-ci-e2e-test resourceVersion: "19489" uid: e22fe3d9-dcf2-4682-b066-9729c760920a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-c6676b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19475" uid: 1fb864be-f53f-4eb1-b1f9-8958d5ca9910 kind: Event lastTimestamp: "2026-04-24T14:45:10Z" message: Scaled up replica set isvc-secondary-c6676b-predictor-6df7fcb885 from 0 to 1 metadata: creationTimestamp: "2026-04-24T14:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:45:10Z" name: isvc-secondary-c6676b-predictor.18a95233b63b54e2 namespace: kserve-ci-e2e-test resourceVersion: "19480" uid: ad73f8c0-df7b-4baf-8766-c1d70f175f60 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-04-24T14:45:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-c6676b namespace: kserve-ci-e2e-test resourceVersion: "19467" uid: 190c3298-652f-430b-84ba-b99bf06e8e1c kind: Event lastTimestamp: "2026-04-24T14:45:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T14:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:45:19Z" name: isvc-secondary-c6676b.18a95233b615c0bc namespace: kserve-ci-e2e-test resourceVersion: "19593" uid: 100eba9e-670a-434a-b4d1-885819c29a5f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-c6676b namespace: kserve-ci-e2e-test resourceVersion: "19467" uid: 190c3298-652f-430b-84ba-b99bf06e8e1c kind: Event lastTimestamp: "2026-04-24T14:45:10Z" message: 'Failed to update status for InferenceService "isvc-secondary-c6676b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-c6676b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T14:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:45:10Z" name: isvc-secondary-c6676b.18a95233b68d3fc5 namespace: kserve-ci-e2e-test resourceVersion: "19482" uid: a0c282e9-c085-4c13-93af-ee35543fee20 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-c6676b namespace: kserve-ci-e2e-test resourceVersion: "19467" uid: 190c3298-652f-430b-84ba-b99bf06e8e1c kind: Event lastTimestamp: "2026-04-24T14:45:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-c6676b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T14:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:45:10Z" name: isvc-secondary-c6676b.18a95233b68da720 namespace: kserve-ci-e2e-test resourceVersion: "19483" uid: df6b3a3a-a707-47f8-b1cd-8dd121c89a82 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T14:36:23.813324Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: b0829893-cbda-4b55-88b3-077ae1ea9f09 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz to ip-10-0-143-92.ec2.internal metadata: creationTimestamp: "2026-04-24T14:36:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T14:36:23Z" name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz.18a951b9315cc851 namespace: kserve-ci-e2e-test resourceVersion: "14934" uid: 196a1dfc-0fdb-4eb9-8f77-d7775a444a16 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd58b56dd-ls688 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:24Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz namespace: kserve-ci-e2e-test resourceVersion: "14939" uid: b0829893-cbda-4b55-88b3-077ae1ea9f09 kind: Event lastTimestamp: "2026-04-24T14:36:24Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T14:36:24Z" name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz.18a951b94ad18f1e namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: b3dd20a8-dd7b-4977-8399-d3b17ad86b60 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: b0829893-cbda-4b55-88b3-077ae1ea9f09 kind: Event lastTimestamp: "2026-04-24T14:36:24Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3370fbcd366f44a8710ac5ab5a02712bec0322005baa64fee46cbff50aae7c18" already present on machine metadata: creationTimestamp: "2026-04-24T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:36:24Z" name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz.18a951b94c22c3e9 namespace: kserve-ci-e2e-test resourceVersion: "14965" uid: 0bb4270f-3b6c-4566-9bcd-6b0f167f2eea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: b0829893-cbda-4b55-88b3-077ae1ea9f09 kind: Event lastTimestamp: "2026-04-24T14:36:24Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:36:24Z" name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz.18a951b94f979c39 namespace: kserve-ci-e2e-test resourceVersion: "14969" uid: 7df5596a-a07d-4687-a9ce-68fbd0e928ca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: b0829893-cbda-4b55-88b3-077ae1ea9f09 kind: Event lastTimestamp: "2026-04-24T14:36:24Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:36:24Z" name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz.18a951b95086570b namespace: kserve-ci-e2e-test resourceVersion: "14970" uid: f624710f-580e-4417-9200-b9391369e829 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: b0829893-cbda-4b55-88b3-077ae1ea9f09 kind: Event lastTimestamp: "2026-04-24T14:36:28Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-04-24T14:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:36:28Z" name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz.18a951ba3bee2a97 namespace: kserve-ci-e2e-test resourceVersion: "15017" uid: 42a6dcfe-522d-4fc0-b242-2b4e2eca49c4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: b0829893-cbda-4b55-88b3-077ae1ea9f09 kind: Event lastTimestamp: "2026-04-24T14:36:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T14:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:36:28Z" name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz.18a951ba3ed43a40 namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: d162bdd7-15a5-40e6-aae9-e92eb41e51de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: b0829893-cbda-4b55-88b3-077ae1ea9f09 kind: Event lastTimestamp: "2026-04-24T14:36:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T14:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:36:28Z" name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz.18a951ba3f4807db namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: 0dc453e0-ac45-4a4f-a8dc-c01924e69aab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T14:36:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: b0829893-cbda-4b55-88b3-077ae1ea9f09 kind: Event lastTimestamp: "2026-04-24T14:37:30Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T14:36:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:37:30Z" name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz.18a951ba7819637e namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: b3bef6c2-a466-4327-b66f-b4fddf2bd98b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: b0829893-cbda-4b55-88b3-077ae1ea9f09 kind: Event lastTimestamp: "2026-04-24T14:38:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T14:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:38:04Z" name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz.18a951d08672df58 namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: 2a8c243f-facc-4c62-9dd0-04f00b5d30c4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf namespace: kserve-ci-e2e-test resourceVersion: "14900" uid: 0f9dc13a-3b47-499c-bbb2-f718f72a8dd2 kind: Event lastTimestamp: "2026-04-24T14:36:23Z" message: 'Created pod: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf-9fqpz' metadata: creationTimestamp: "2026-04-24T14:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:36:23Z" name: isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf.18a951b93114584d namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: cd9fc7a1-90a9-4951-b1ff-019b8b5d6b81 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-a4202-predictor namespace: kserve-ci-e2e-test resourceVersion: "14897" uid: 4087d316-93f5-40bc-939c-3a721aaca2d9 kind: Event lastTimestamp: "2026-04-24T14:36:23Z" message: Scaled up replica set isvc-sklearn-graph-raw-a4202-predictor-dfbfb6fcf from 0 to 1 metadata: creationTimestamp: "2026-04-24T14:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:36:23Z" name: isvc-sklearn-graph-raw-a4202-predictor.18a951b929375815 namespace: kserve-ci-e2e-test resourceVersion: "14904" uid: 0785aec0-34c8-4980-95db-67478c332ccc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-a4202-predictor namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 761ed50f-9deb-4a66-868e-b895101d4868 kind: Event lastTimestamp: "2026-04-24T14:36:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:36:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:36:38Z" name: isvc-sklearn-graph-raw-a4202-predictor.18a951bca828b110 namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: b540b7e0-357c-4bf9-afa9-f91c3a885306 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-a4202-predictor namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 761ed50f-9deb-4a66-868e-b895101d4868 kind: Event lastTimestamp: "2026-04-24T14:36:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:36:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:36:38Z" name: isvc-sklearn-graph-raw-a4202-predictor.18a951bca89acebc namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: 31ac6daa-50d9-4f63-8a53-af92d5d2b99a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T14:36:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-a4202-predictor namespace: kserve-ci-e2e-test resourceVersion: "15106" uid: 761ed50f-9deb-4a66-868e-b895101d4868 kind: Event lastTimestamp: "2026-04-24T14:37:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T14:36:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:37:53Z" name: isvc-sklearn-graph-raw-a4202-predictor.18a951c0275ab555 namespace: kserve-ci-e2e-test resourceVersion: "15703" uid: ecfda573-3fcb-4e5e-971c-76bee0b90aa3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T14:36:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-a4202-predictor namespace: kserve-ci-e2e-test resourceVersion: "15106" uid: 761ed50f-9deb-4a66-868e-b895101d4868 kind: Event lastTimestamp: "2026-04-24T14:37:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T14:36:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:37:53Z" name: isvc-sklearn-graph-raw-a4202-predictor.18a951c027bbe080 namespace: kserve-ci-e2e-test resourceVersion: "15706" uid: e5fafd4f-3985-4d45-b7e8-f9cfea6c092c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-24T14:36:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-a4202 namespace: kserve-ci-e2e-test resourceVersion: "14875" uid: 9d4bc581-1b48-4558-886b-2da9a11dd85b kind: Event lastTimestamp: "2026-04-24T14:37:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T14:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:37:40Z" name: isvc-sklearn-graph-raw-a4202.18a951b9290dbabe namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 4ab1ce8a-eba4-4e49-aa43-e2cc510d3b73 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-a4202 namespace: kserve-ci-e2e-test resourceVersion: "14875" uid: 9d4bc581-1b48-4558-886b-2da9a11dd85b kind: Event lastTimestamp: "2026-04-24T14:36:23Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-a4202": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-a4202": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T14:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:36:23Z" name: isvc-sklearn-graph-raw-a4202.18a951b929d50aca namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: 9f349fe2-eff7-47ca-90e2-6b7ecc9e8b2d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:37:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-a4202 namespace: kserve-ci-e2e-test resourceVersion: "15585" uid: 9d4bc581-1b48-4558-886b-2da9a11dd85b kind: Event lastTimestamp: "2026-04-24T14:37:40Z" message: InferenceService [isvc-sklearn-graph-raw-a4202] is Ready metadata: creationTimestamp: "2026-04-24T14:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:37:40Z" name: isvc-sklearn-graph-raw-a4202.18a951cb04c3afb7 namespace: kserve-ci-e2e-test resourceVersion: "15586" uid: 0713b92d-a9cc-421e-807b-c3443a82096f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T14:38:04.063345Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8 namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: e3c95b37-9d70-4ff8-bf3b-9fc158bf3b2c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8 to ip-10-0-143-92.ec2.internal metadata: creationTimestamp: "2026-04-24T14:38:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T14:38:04Z" name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8.18a951d088baa1b1 namespace: kserve-ci-e2e-test resourceVersion: "15879" uid: 773d8eb4-45fa-455e-9a99-c65ae47c3576 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd58b56dd-ls688 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8 namespace: kserve-ci-e2e-test resourceVersion: "15885" uid: e3c95b37-9d70-4ff8-bf3b-9fc158bf3b2c kind: Event lastTimestamp: "2026-04-24T14:38:04Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T14:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T14:38:04Z" name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8.18a951d0a271e657 namespace: kserve-ci-e2e-test resourceVersion: "15926" uid: 9ccbcfcc-4e0e-479e-a84d-c49ae64e9465 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8 namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: e3c95b37-9d70-4ff8-bf3b-9fc158bf3b2c kind: Event lastTimestamp: "2026-04-24T14:38:04Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3370fbcd366f44a8710ac5ab5a02712bec0322005baa64fee46cbff50aae7c18" already present on machine metadata: creationTimestamp: "2026-04-24T14:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:38:04Z" name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8.18a951d0a3d323c7 namespace: kserve-ci-e2e-test resourceVersion: "15928" uid: 4872373b-5fe9-4dd3-b725-6d5528d08fa2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8 namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: e3c95b37-9d70-4ff8-bf3b-9fc158bf3b2c kind: Event lastTimestamp: "2026-04-24T14:38:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T14:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:38:04Z" name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8.18a951d0a7491850 namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: ebcfe1c4-1863-46fd-9ea4-40ffa234396c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8 namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: e3c95b37-9d70-4ff8-bf3b-9fc158bf3b2c kind: Event lastTimestamp: "2026-04-24T14:38:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T14:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:38:04Z" name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8.18a951d0a7cc1a93 namespace: kserve-ci-e2e-test resourceVersion: "15935" uid: f56e2037-33c2-4bdb-bba7-a6bc6fb9bc16 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8 namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: e3c95b37-9d70-4ff8-bf3b-9fc158bf3b2c kind: Event lastTimestamp: "2026-04-24T14:38:08Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-04-24T14:38:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:38:08Z" name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8.18a951d1976de277 namespace: kserve-ci-e2e-test resourceVersion: "15979" uid: 4e7867cf-d429-4b3c-bb84-c16243406e97 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8 namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: e3c95b37-9d70-4ff8-bf3b-9fc158bf3b2c kind: Event lastTimestamp: "2026-04-24T14:38:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T14:38:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:38:08Z" name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8.18a951d19b651526 namespace: kserve-ci-e2e-test resourceVersion: "15988" uid: 037d10c5-47ee-4397-b2ea-3d00a2c063ea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8 namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: e3c95b37-9d70-4ff8-bf3b-9fc158bf3b2c kind: Event lastTimestamp: "2026-04-24T14:38:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T14:38:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:38:08Z" name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8.18a951d19be454fe namespace: kserve-ci-e2e-test resourceVersion: "15991" uid: ca597d3e-b59f-4f75-a37a-41d0df871e8b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T14:38:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8 namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: e3c95b37-9d70-4ff8-bf3b-9fc158bf3b2c kind: Event lastTimestamp: "2026-04-24T14:39:10Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T14:38:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:10Z" name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8.18a951d1d3c0db40 namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: 08108fd2-12c0-4f52-86a1-08a2769171b9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8 namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: e3c95b37-9d70-4ff8-bf3b-9fc158bf3b2c kind: Event lastTimestamp: "2026-04-24T14:39:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T14:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:44Z" name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8.18a951e7e37de4e7 namespace: kserve-ci-e2e-test resourceVersion: "16835" uid: f40f1940-1972-4d9e-811c-6ef52c770979 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 9541981e-798d-4065-ac2a-daeb66185765 kind: Event lastTimestamp: "2026-04-24T14:38:04Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc-8b2t8' metadata: creationTimestamp: "2026-04-24T14:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:38:04Z" name: isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc.18a951d088291d3d namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: ca73bb99-8461-49c5-8906-07d79820489f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-29d74-predictor namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: f94bce45-8b18-451a-b126-f2c218ebc30a kind: Event lastTimestamp: "2026-04-24T14:38:03Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-29d74-predictor-7895bbd4dc from 0 to 1 metadata: creationTimestamp: "2026-04-24T14:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:38:03Z" name: isvc-sklearn-graph-raw-hpa-29d74-predictor.18a951d08360454d namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: 4572ff48-59ea-4537-a4ca-88b35e849797 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T14:38:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-29d74-predictor namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: aae22d2b-19ca-4e29-817a-f7922089229e kind: Event lastTimestamp: "2026-04-24T14:38:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:38:49Z" name: isvc-sklearn-graph-raw-hpa-29d74-predictor.18a951d401ddb29c namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: a30af6ea-9f55-4a73-89b1-0cf4c5521d3f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T14:38:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-29d74-predictor namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: aae22d2b-19ca-4e29-817a-f7922089229e kind: Event lastTimestamp: "2026-04-24T14:38:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:38:49Z" name: isvc-sklearn-graph-raw-hpa-29d74-predictor.18a951d40246b2f6 namespace: kserve-ci-e2e-test resourceVersion: "16301" uid: 99bfcef0-d659-43ff-855b-7dba439fe72e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T14:39:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-29d74-predictor namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: aae22d2b-19ca-4e29-817a-f7922089229e kind: Event lastTimestamp: "2026-04-24T14:39:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T14:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:39:34Z" name: isvc-sklearn-graph-raw-hpa-29d74-predictor.18a951de7f935123 namespace: kserve-ci-e2e-test resourceVersion: "16644" uid: 09a364bc-64d6-4576-b89a-532edaf5dba4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T14:39:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-29d74-predictor namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: aae22d2b-19ca-4e29-817a-f7922089229e kind: Event lastTimestamp: "2026-04-24T14:39:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T14:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:39:34Z" name: isvc-sklearn-graph-raw-hpa-29d74-predictor.18a951de80129015 namespace: kserve-ci-e2e-test resourceVersion: "16646" uid: 97007a9f-feb9-4a83-a32b-b5d44ba2372e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-24T14:38:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-29d74 namespace: kserve-ci-e2e-test resourceVersion: "15828" uid: 41aea971-529a-4201-92b5-a6848ce88c8c kind: Event lastTimestamp: "2026-04-24T14:39:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T14:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:39:20Z" name: isvc-sklearn-graph-raw-hpa-29d74.18a951d08329fadd namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: cfc93258-093f-484b-91d7-b5c66d8bf4af reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-29d74 namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 41aea971-529a-4201-92b5-a6848ce88c8c kind: Event lastTimestamp: "2026-04-24T14:39:20Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-29d74] is Ready metadata: creationTimestamp: "2026-04-24T14:39:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:39:20Z" name: isvc-sklearn-graph-raw-hpa-29d74.18a951e26011ed60 namespace: kserve-ci-e2e-test resourceVersion: "16549" uid: 027b08ad-1747-46f1-83a6-d27fb88fcf62 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T14:41:19.693727Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778 namespace: kserve-ci-e2e-test resourceVersion: "17678" uid: a3ebb39c-aa31-41ed-9dea-5d2234505328 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778 to ip-10-0-143-92.ec2.internal metadata: creationTimestamp: "2026-04-24T14:41:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T14:41:19Z" name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778.18a951fe1535562d namespace: kserve-ci-e2e-test resourceVersion: "17683" uid: a3ff5771-9337-440f-b761-61d6b0fa2622 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd58b56dd-ls688 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:41:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778 namespace: kserve-ci-e2e-test resourceVersion: "17687" uid: a3ebb39c-aa31-41ed-9dea-5d2234505328 kind: Event lastTimestamp: "2026-04-24T14:41:20Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T14:41:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T14:41:20Z" name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778.18a951fe2e25479d namespace: kserve-ci-e2e-test resourceVersion: "17716" uid: 82128f2f-cdb6-4815-b491-ca3c8a5fa04c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:41:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778 namespace: kserve-ci-e2e-test resourceVersion: "17682" uid: a3ebb39c-aa31-41ed-9dea-5d2234505328 kind: Event lastTimestamp: "2026-04-24T14:41:20Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3370fbcd366f44a8710ac5ab5a02712bec0322005baa64fee46cbff50aae7c18" already present on machine metadata: creationTimestamp: "2026-04-24T14:41:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:41:20Z" name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778.18a951fe2f560995 namespace: kserve-ci-e2e-test resourceVersion: "17718" uid: 518c6822-3b0e-4989-8d95-60aee746930d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:41:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778 namespace: kserve-ci-e2e-test resourceVersion: "17682" uid: a3ebb39c-aa31-41ed-9dea-5d2234505328 kind: Event lastTimestamp: "2026-04-24T14:41:20Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T14:41:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:41:20Z" name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778.18a951fe325dcecf namespace: kserve-ci-e2e-test resourceVersion: "17719" uid: 60fcba0a-9caf-4e77-919d-c7bd91565ac8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:41:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778 namespace: kserve-ci-e2e-test resourceVersion: "17682" uid: a3ebb39c-aa31-41ed-9dea-5d2234505328 kind: Event lastTimestamp: "2026-04-24T14:41:20Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T14:41:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:41:20Z" name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778.18a951fe32daae9f namespace: kserve-ci-e2e-test resourceVersion: "17720" uid: 3a7aa810-322b-4da7-ab95-da4f93257b97 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:41:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778 namespace: kserve-ci-e2e-test resourceVersion: "17682" uid: a3ebb39c-aa31-41ed-9dea-5d2234505328 kind: Event lastTimestamp: "2026-04-24T14:41:24Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-04-24T14:41:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:41:24Z" name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778.18a951ff222098ac namespace: kserve-ci-e2e-test resourceVersion: "17761" uid: db6a20bb-0837-4560-8a1d-f3dd5bb69c10 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:41:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778 namespace: kserve-ci-e2e-test resourceVersion: "17682" uid: a3ebb39c-aa31-41ed-9dea-5d2234505328 kind: Event lastTimestamp: "2026-04-24T14:41:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T14:41:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:41:24Z" name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778.18a951ff261fc066 namespace: kserve-ci-e2e-test resourceVersion: "17765" uid: 4c90b8c4-6379-425a-a7ce-2e05c8d5ec84 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:41:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778 namespace: kserve-ci-e2e-test resourceVersion: "17682" uid: a3ebb39c-aa31-41ed-9dea-5d2234505328 kind: Event lastTimestamp: "2026-04-24T14:41:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T14:41:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:41:24Z" name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778.18a951ff269ba892 namespace: kserve-ci-e2e-test resourceVersion: "17766" uid: 33012271-b9e8-4a2a-ba5d-43f6ef98f877 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 18 eventTime: null firstTimestamp: "2026-04-24T14:41:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778 namespace: kserve-ci-e2e-test resourceVersion: "17682" uid: a3ebb39c-aa31-41ed-9dea-5d2234505328 kind: Event lastTimestamp: "2026-04-24T14:43:58Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T14:41:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:43:58Z" name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778.18a951ff5e0bdce7 namespace: kserve-ci-e2e-test resourceVersion: "18938" uid: 751cd053-700f-4924-b1ae-a25d3449a538 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:43:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778 namespace: kserve-ci-e2e-test resourceVersion: "17682" uid: a3ebb39c-aa31-41ed-9dea-5d2234505328 kind: Event lastTimestamp: "2026-04-24T14:43:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T14:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:43:49Z" name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778.18a952210b818d4a namespace: kserve-ci-e2e-test resourceVersion: "18834" uid: 2c432f53-328d-4d44-aabf-2aa07dd89693 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:41:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c namespace: kserve-ci-e2e-test resourceVersion: "17653" uid: 1acbe1a2-9b1e-4e46-bff6-577f5579aaca kind: Event lastTimestamp: "2026-04-24T14:41:19Z" message: 'Created pod: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c-xn778' metadata: creationTimestamp: "2026-04-24T14:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:41:19Z" name: isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c.18a951fe149a3006 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 16538eec-14e2-4334-b7d0-2ce1484004df reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:41:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-2a9b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17646" uid: 3ec61834-1803-4ca8-b6e9-5dc5b10a487f kind: Event lastTimestamp: "2026-04-24T14:41:19Z" message: Scaled up replica set isvc-sklearn-scale-raw-2a9b2-predictor-5f6f8cb7c from 0 to 1 metadata: creationTimestamp: "2026-04-24T14:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:41:19Z" name: isvc-sklearn-scale-raw-2a9b2-predictor.18a951fe10e24cbd namespace: kserve-ci-e2e-test resourceVersion: "17658" uid: 34ab9299-0ed9-4102-8d5c-d24cc4afc2d3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T14:41:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-2a9b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17651" uid: 6b8e3442-cea7-4bc2-a7cc-ad57140748a8 kind: Event lastTimestamp: "2026-04-24T14:42:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:41:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:42:04Z" name: isvc-sklearn-scale-raw-2a9b2-predictor.18a952018f86eeca namespace: kserve-ci-e2e-test resourceVersion: "18058" uid: 4cf77cc2-ec9d-442c-84bc-4d4dd17a9808 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T14:41:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-2a9b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17651" uid: 6b8e3442-cea7-4bc2-a7cc-ad57140748a8 kind: Event lastTimestamp: "2026-04-24T14:42:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:41:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:42:04Z" name: isvc-sklearn-scale-raw-2a9b2-predictor.18a952018fef7c75 namespace: kserve-ci-e2e-test resourceVersion: "18059" uid: 1930dd0e-eda2-4954-b596-46fd8cd05295 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T14:42:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-2a9b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17845" uid: 6b8e3442-cea7-4bc2-a7cc-ad57140748a8 kind: Event lastTimestamp: "2026-04-24T14:43:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T14:42:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:43:49Z" name: isvc-sklearn-scale-raw-2a9b2-predictor.18a9520c0ca36413 namespace: kserve-ci-e2e-test resourceVersion: "18808" uid: 34d5f648-9393-4b55-8c1f-6dc006b55e11 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T14:42:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-2a9b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17845" uid: 6b8e3442-cea7-4bc2-a7cc-ad57140748a8 kind: Event lastTimestamp: "2026-04-24T14:43:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T14:42:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:43:49Z" name: isvc-sklearn-scale-raw-2a9b2-predictor.18a9520c0d183578 namespace: kserve-ci-e2e-test resourceVersion: "18810" uid: 6c2248e5-73f6-4e36-9eec-00e3a07106bd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-24T14:41:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-2a9b2 namespace: kserve-ci-e2e-test resourceVersion: "17638" uid: 8d3cde0f-8f3c-4fa8-9819-54a9205c38a0 kind: Event lastTimestamp: "2026-04-24T14:43:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T14:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:43:48Z" name: isvc-sklearn-scale-raw-2a9b2.18a951fe10944e62 namespace: kserve-ci-e2e-test resourceVersion: "18796" uid: 77abf0ba-2dfa-4ef8-8ba2-eed62ddd7aec reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T14:41:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-2a9b2 namespace: kserve-ci-e2e-test resourceVersion: "17638" uid: 8d3cde0f-8f3c-4fa8-9819-54a9205c38a0 kind: Event lastTimestamp: "2026-04-24T14:41:19Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-2a9b2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-2a9b2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T14:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:41:19Z" name: isvc-sklearn-scale-raw-2a9b2.18a951fe115aa4e5 namespace: kserve-ci-e2e-test resourceVersion: "17689" uid: ed52e01f-2f2f-4722-8826-209808fdcb2b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:41:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-2a9b2 namespace: kserve-ci-e2e-test resourceVersion: "17656" uid: 8d3cde0f-8f3c-4fa8-9819-54a9205c38a0 kind: Event lastTimestamp: "2026-04-24T14:41:19Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-2a9b2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T14:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:41:19Z" name: isvc-sklearn-scale-raw-2a9b2.18a951fe16f8fc18 namespace: kserve-ci-e2e-test resourceVersion: "17690" uid: 04b4aa29-4be9-412c-9b25-c2fa5d8aef03 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:43:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-2a9b2 namespace: kserve-ci-e2e-test resourceVersion: "18792" uid: 8d3cde0f-8f3c-4fa8-9819-54a9205c38a0 kind: Event lastTimestamp: "2026-04-24T14:43:48Z" message: InferenceService [isvc-sklearn-scale-raw-2a9b2] is Ready metadata: creationTimestamp: "2026-04-24T14:43:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:43:48Z" name: isvc-sklearn-scale-raw-2a9b2.18a95220ab673ce7 namespace: kserve-ci-e2e-test resourceVersion: "18793" uid: 2e89f6d9-1993-49ce-b8d4-bba0a5879b96 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T14:36:23.850125Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px namespace: kserve-ci-e2e-test resourceVersion: "14938" uid: c0388cf9-12c9-4ee2-ac85-53ef90abe77c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px to ip-10-0-143-92.ec2.internal metadata: creationTimestamp: "2026-04-24T14:36:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T14:36:23Z" name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px.18a951b9338e3e1a namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 1f2a17ce-ddc2-4e7c-943c-d16073b65981 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd58b56dd-ls688 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:24Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: c0388cf9-12c9-4ee2-ac85-53ef90abe77c kind: Event lastTimestamp: "2026-04-24T14:36:24Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T14:36:24Z" name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px.18a951b94d3555c4 namespace: kserve-ci-e2e-test resourceVersion: "14966" uid: d781c3a6-1ffa-4459-9433-971a7d5c6a81 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: c0388cf9-12c9-4ee2-ac85-53ef90abe77c kind: Event lastTimestamp: "2026-04-24T14:36:24Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3370fbcd366f44a8710ac5ab5a02712bec0322005baa64fee46cbff50aae7c18" already present on machine metadata: creationTimestamp: "2026-04-24T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:36:24Z" name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px.18a951b94e9417c4 namespace: kserve-ci-e2e-test resourceVersion: "14968" uid: b4e2fd8c-6183-44ae-a3a7-e4088488062b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: c0388cf9-12c9-4ee2-ac85-53ef90abe77c kind: Event lastTimestamp: "2026-04-24T14:36:24Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:36:24Z" name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px.18a951b951d62f24 namespace: kserve-ci-e2e-test resourceVersion: "14972" uid: 931395ee-0fc5-4aa7-a8c6-7d7f5d328e53 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: c0388cf9-12c9-4ee2-ac85-53ef90abe77c kind: Event lastTimestamp: "2026-04-24T14:36:24Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:36:24Z" name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px.18a951b9524cb0ef namespace: kserve-ci-e2e-test resourceVersion: "14973" uid: f641e7ec-625d-4db0-9d6d-a149a055e7b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: c0388cf9-12c9-4ee2-ac85-53ef90abe77c kind: Event lastTimestamp: "2026-04-24T14:36:28Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-24T14:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:36:28Z" name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px.18a951ba3c053fe9 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: bd02573f-b2f6-4d63-8a84-4d723ccf3a81 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: c0388cf9-12c9-4ee2-ac85-53ef90abe77c kind: Event lastTimestamp: "2026-04-24T14:36:47Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.329s (19.329s including waiting). Image size: 1306417403 bytes.' metadata: creationTimestamp: "2026-04-24T14:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:36:47Z" name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px.18a951bebc265d71 namespace: kserve-ci-e2e-test resourceVersion: "15175" uid: 5fdea03a-439b-470f-9436-522c00ddb743 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: c0388cf9-12c9-4ee2-ac85-53ef90abe77c kind: Event lastTimestamp: "2026-04-24T14:36:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T14:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:36:47Z" name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px.18a951bec02387e8 namespace: kserve-ci-e2e-test resourceVersion: "15177" uid: 1a21e24c-9549-4c50-ae89-cc5d8b0055e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: c0388cf9-12c9-4ee2-ac85-53ef90abe77c kind: Event lastTimestamp: "2026-04-24T14:36:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T14:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:36:47Z" name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px.18a951bec09b21ce namespace: kserve-ci-e2e-test resourceVersion: "15178" uid: d943fc38-4f42-40ea-872a-63f8464cba73 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T14:36:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: c0388cf9-12c9-4ee2-ac85-53ef90abe77c kind: Event lastTimestamp: "2026-04-24T14:37:39Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T14:36:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:37:39Z" name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px.18a951bee8a6abb9 namespace: kserve-ci-e2e-test resourceVersion: "15571" uid: f0355e78-c00f-4203-a8ac-c096e8c8c097 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: c0388cf9-12c9-4ee2-ac85-53ef90abe77c kind: Event lastTimestamp: "2026-04-24T14:38:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T14:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:38:04Z" name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px.18a951d0930718ca namespace: kserve-ci-e2e-test resourceVersion: "15920" uid: 735319c1-d218-4890-b3d1-e6954ca16e09 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5 namespace: kserve-ci-e2e-test resourceVersion: "14923" uid: 3df738ff-db41-48cd-853d-245ce7f7ced6 kind: Event lastTimestamp: "2026-04-24T14:36:23Z" message: 'Created pod: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5-5m8px' metadata: creationTimestamp: "2026-04-24T14:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:36:23Z" name: isvc-xgboost-graph-raw-a4202-predictor-548565d6f5.18a951b9330164d5 namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: bc4aee54-d681-4e66-a8e6-5916fabb72c5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-a4202-predictor namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 943b45cb-d594-4f93-a8a6-06684f51d08b kind: Event lastTimestamp: "2026-04-24T14:36:23Z" message: Scaled up replica set isvc-xgboost-graph-raw-a4202-predictor-548565d6f5 from 0 to 1 metadata: creationTimestamp: "2026-04-24T14:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:36:23Z" name: isvc-xgboost-graph-raw-a4202-predictor.18a951b92faf8cf7 namespace: kserve-ci-e2e-test resourceVersion: "14925" uid: dd07173e-8dfa-412f-9a2c-d531c19e9bf3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T14:36:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-a4202-predictor namespace: kserve-ci-e2e-test resourceVersion: "14922" uid: ad4ed898-d9a7-46ca-a31f-8ac44fa905a0 kind: Event lastTimestamp: "2026-04-24T14:37:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:36:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:37:23Z" name: isvc-xgboost-graph-raw-a4202-predictor.18a951bcae3a15e6 namespace: kserve-ci-e2e-test resourceVersion: "15457" uid: a7abdca1-c961-492d-a996-16351a16108e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T14:36:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-a4202-predictor namespace: kserve-ci-e2e-test resourceVersion: "14922" uid: ad4ed898-d9a7-46ca-a31f-8ac44fa905a0 kind: Event lastTimestamp: "2026-04-24T14:37:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:36:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:37:23Z" name: isvc-xgboost-graph-raw-a4202-predictor.18a951bcaea66b1a namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: 4695d133-9b50-46ea-bd95-060eafd776aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T14:37:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-a4202-predictor namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: ad4ed898-d9a7-46ca-a31f-8ac44fa905a0 kind: Event lastTimestamp: "2026-04-24T14:37:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T14:37:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:37:53Z" name: isvc-xgboost-graph-raw-a4202-predictor.18a951caa9a8415d namespace: kserve-ci-e2e-test resourceVersion: "15716" uid: 20da77ff-3db3-45a7-b317-37360987677a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T14:37:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-a4202-predictor namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: ad4ed898-d9a7-46ca-a31f-8ac44fa905a0 kind: Event lastTimestamp: "2026-04-24T14:37:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T14:37:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:37:53Z" name: isvc-xgboost-graph-raw-a4202-predictor.18a951caaa126875 namespace: kserve-ci-e2e-test resourceVersion: "15717" uid: 3ed7255c-243e-4059-931a-5331fc2123a9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T14:36:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-a4202 namespace: kserve-ci-e2e-test resourceVersion: "14912" uid: 10b1b4c5-3d86-4470-86d8-c0087427c994 kind: Event lastTimestamp: "2026-04-24T14:37:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T14:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:37:49Z" name: isvc-xgboost-graph-raw-a4202.18a951b92f751641 namespace: kserve-ci-e2e-test resourceVersion: "15653" uid: 01ab7d9c-7571-4924-812b-7a1a4a884182 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:36:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-a4202 namespace: kserve-ci-e2e-test resourceVersion: "14912" uid: 10b1b4c5-3d86-4470-86d8-c0087427c994 kind: Event lastTimestamp: "2026-04-24T14:36:23Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-a4202": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-a4202": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T14:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:36:23Z" name: isvc-xgboost-graph-raw-a4202.18a951b9301dc939 namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: 9f312bf3-bf22-443e-8253-4e8caceb32aa reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:37:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-a4202 namespace: kserve-ci-e2e-test resourceVersion: "15651" uid: 10b1b4c5-3d86-4470-86d8-c0087427c994 kind: Event lastTimestamp: "2026-04-24T14:37:49Z" message: InferenceService [isvc-xgboost-graph-raw-a4202] is Ready metadata: creationTimestamp: "2026-04-24T14:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:37:49Z" name: isvc-xgboost-graph-raw-a4202.18a951cd21a3f6ca namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: 3a1bbf2e-c8cb-4c7a-bc7f-4f061ad72f52 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T14:38:04.119913Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: a5ada30a-c8f3-421f-b2a7-e901ed97b1d7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t to ip-10-0-143-92.ec2.internal metadata: creationTimestamp: "2026-04-24T14:38:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T14:38:04Z" name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t.18a951d08c19e3d1 namespace: kserve-ci-e2e-test resourceVersion: "15896" uid: 7012c149-e937-44d4-9669-6307ac25a23d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd58b56dd-ls688 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t namespace: kserve-ci-e2e-test resourceVersion: "15902" uid: a5ada30a-c8f3-421f-b2a7-e901ed97b1d7 kind: Event lastTimestamp: "2026-04-24T14:38:04Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T14:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T14:38:04Z" name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t.18a951d0a60dd890 namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: d5baf3d5-2e80-40de-91ce-9914df81dbc3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: a5ada30a-c8f3-421f-b2a7-e901ed97b1d7 kind: Event lastTimestamp: "2026-04-24T14:38:04Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3370fbcd366f44a8710ac5ab5a02712bec0322005baa64fee46cbff50aae7c18" already present on machine metadata: creationTimestamp: "2026-04-24T14:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:38:04Z" name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t.18a951d0a7640394 namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: 619637f6-f5c8-40c7-82fc-1ad1dac54a59 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: a5ada30a-c8f3-421f-b2a7-e901ed97b1d7 kind: Event lastTimestamp: "2026-04-24T14:38:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T14:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:38:04Z" name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t.18a951d0aacf8041 namespace: kserve-ci-e2e-test resourceVersion: "15937" uid: 26b6ff3e-c254-4c43-915a-9973a480e5ff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: a5ada30a-c8f3-421f-b2a7-e901ed97b1d7 kind: Event lastTimestamp: "2026-04-24T14:38:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T14:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:38:04Z" name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t.18a951d0ab499c7e namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 0cee5b87-8b8c-4840-83c5-6efe1b6cdbb5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: a5ada30a-c8f3-421f-b2a7-e901ed97b1d7 kind: Event lastTimestamp: "2026-04-24T14:38:08Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-24T14:38:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:38:08Z" name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t.18a951d19785fe1f namespace: kserve-ci-e2e-test resourceVersion: "15980" uid: a15391bc-c7d6-4aeb-be49-90750e9447fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: a5ada30a-c8f3-421f-b2a7-e901ed97b1d7 kind: Event lastTimestamp: "2026-04-24T14:38:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T14:38:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:38:08Z" name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t.18a951d19b572a4e namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: aaac56fb-7549-4e61-9057-86dae4ff653e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: a5ada30a-c8f3-421f-b2a7-e901ed97b1d7 kind: Event lastTimestamp: "2026-04-24T14:38:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T14:38:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:38:08Z" name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t.18a951d19bd748ab namespace: kserve-ci-e2e-test resourceVersion: "15990" uid: 714ce47c-30e6-4854-8f7a-bcb9ef1cf78d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T14:38:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: a5ada30a-c8f3-421f-b2a7-e901ed97b1d7 kind: Event lastTimestamp: "2026-04-24T14:39:00Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T14:38:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:00Z" name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t.18a951d1d3d436f0 namespace: kserve-ci-e2e-test resourceVersion: "16384" uid: 656710e3-a8a4-4f92-9ceb-6ba6a806eeae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: a5ada30a-c8f3-421f-b2a7-e901ed97b1d7 kind: Event lastTimestamp: "2026-04-24T14:39:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T14:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:44Z" name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t.18a951e7e07281b1 namespace: kserve-ci-e2e-test resourceVersion: "16812" uid: 2ebf5aa2-ae49-451e-8a51-48126b070da4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c namespace: kserve-ci-e2e-test resourceVersion: "15875" uid: 4c75acc5-c190-4016-8b43-d3e9cd9dc8da kind: Event lastTimestamp: "2026-04-24T14:38:04Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c-k6j4t' metadata: creationTimestamp: "2026-04-24T14:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:38:04Z" name: isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c.18a951d08b894240 namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: 72ee9164-fc67-45af-9542-c3d400b62cd6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-29d74-predictor namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: e7dbcacd-8622-44ce-8349-faed600552de kind: Event lastTimestamp: "2026-04-24T14:38:04Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-29d74-predictor-5bf5b4664c from 0 to 1 metadata: creationTimestamp: "2026-04-24T14:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:38:04Z" name: isvc-xgboost-graph-raw-hpa-29d74-predictor.18a951d088acd340 namespace: kserve-ci-e2e-test resourceVersion: "15878" uid: bd85f264-11d3-4c4c-9a1b-2a774572f179 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-29d74-predictor namespace: kserve-ci-e2e-test resourceVersion: "15871" uid: 0b065e89-9101-474d-8a2e-a4990a7f88c8 kind: Event lastTimestamp: "2026-04-24T14:38:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:38:19Z" name: isvc-xgboost-graph-raw-hpa-29d74-predictor.18a951d4073b0d44 namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: 1aef7c80-f2d6-4818-bfb3-52456dc6bfb4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-29d74-predictor namespace: kserve-ci-e2e-test resourceVersion: "15871" uid: 0b065e89-9101-474d-8a2e-a4990a7f88c8 kind: Event lastTimestamp: "2026-04-24T14:38:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:38:19Z" name: isvc-xgboost-graph-raw-hpa-29d74-predictor.18a951d407ace4dc namespace: kserve-ci-e2e-test resourceVersion: "16081" uid: fd1d3f16-4f38-4181-a0ad-3637da84a419 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T14:38:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-29d74-predictor namespace: kserve-ci-e2e-test resourceVersion: "16080" uid: 0b065e89-9101-474d-8a2e-a4990a7f88c8 kind: Event lastTimestamp: "2026-04-24T14:39:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T14:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:39:34Z" name: isvc-xgboost-graph-raw-hpa-29d74-predictor.18a951d7865701a8 namespace: kserve-ci-e2e-test resourceVersion: "16663" uid: eef508d0-cae4-4269-90d8-39563d39b20f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T14:38:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-29d74-predictor namespace: kserve-ci-e2e-test resourceVersion: "16080" uid: 0b065e89-9101-474d-8a2e-a4990a7f88c8 kind: Event lastTimestamp: "2026-04-24T14:39:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T14:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:39:34Z" name: isvc-xgboost-graph-raw-hpa-29d74-predictor.18a951d786c81ece namespace: kserve-ci-e2e-test resourceVersion: "16667" uid: 38781c7a-1106-4d7c-aa04-0f12db8fb48e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T14:38:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-29d74 namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: 2df92684-2fa1-4d80-afc7-aa83a593bdee kind: Event lastTimestamp: "2026-04-24T14:39:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T14:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:39:10Z" name: isvc-xgboost-graph-raw-hpa-29d74.18a951d0887f8c9d namespace: kserve-ci-e2e-test resourceVersion: "16466" uid: f3e4cf58-551c-4f36-95da-961a4ede625b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-29d74 namespace: kserve-ci-e2e-test resourceVersion: "16464" uid: 2df92684-2fa1-4d80-afc7-aa83a593bdee kind: Event lastTimestamp: "2026-04-24T14:39:10Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-29d74] is Ready metadata: creationTimestamp: "2026-04-24T14:39:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:39:10Z" name: isvc-xgboost-graph-raw-hpa-29d74.18a951e00c87d4a6 namespace: kserve-ci-e2e-test resourceVersion: "16465" uid: df7992b8-a185-41e1-902d-2adc2b6ad371 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T14:39:44.384862Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-cf657-predictor-b44ccb865-rt4mg namespace: kserve-ci-e2e-test resourceVersion: "16832" uid: d7c32f6f-cfaa-4d6f-937a-950a5560eb00 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-cf657-predictor-b44ccb865-rt4mg to ip-10-0-137-95.ec2.internal metadata: creationTimestamp: "2026-04-24T14:39:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T14:39:44Z" name: message-dumper-raw-cf657-predictor-b44ccb865-rt4mg.18a951e7e45b92d5 namespace: kserve-ci-e2e-test resourceVersion: "16843" uid: 7afa4114-e6b8-46fe-9b7f-4ad16da9823b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd58b56dd-ls688 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:44Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-cf657-predictor-b44ccb865-rt4mg namespace: kserve-ci-e2e-test resourceVersion: "16847" uid: d7c32f6f-cfaa-4d6f-937a-950a5560eb00 kind: Event lastTimestamp: "2026-04-24T14:39:44Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T14:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T14:39:44Z" name: message-dumper-raw-cf657-predictor-b44ccb865-rt4mg.18a951e7ec3366c2 namespace: kserve-ci-e2e-test resourceVersion: "16854" uid: 7d7a2584-64a0-47f2-8c32-6bff6dbcb9bc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cf657-predictor-b44ccb865-rt4mg namespace: kserve-ci-e2e-test resourceVersion: "16838" uid: d7c32f6f-cfaa-4d6f-937a-950a5560eb00 kind: Event lastTimestamp: "2026-04-24T14:39:44Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-24T14:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:44Z" name: message-dumper-raw-cf657-predictor-b44ccb865-rt4mg.18a951e7ed839a2a namespace: kserve-ci-e2e-test resourceVersion: "16857" uid: b1179f3b-2550-4e81-a803-3fc9fa635d76 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-95.ec2.internal source: component: kubelet host: ip-10-0-137-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cf657-predictor-b44ccb865-rt4mg namespace: kserve-ci-e2e-test resourceVersion: "16838" uid: d7c32f6f-cfaa-4d6f-937a-950a5560eb00 kind: Event lastTimestamp: "2026-04-24T14:39:45Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.195s (1.195s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-24T14:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:45Z" name: message-dumper-raw-cf657-predictor-b44ccb865-rt4mg.18a951e834c0fe0e namespace: kserve-ci-e2e-test resourceVersion: "16866" uid: 6e2e05f9-0db6-4c98-b43f-3243f97a8df7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-95.ec2.internal source: component: kubelet host: ip-10-0-137-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cf657-predictor-b44ccb865-rt4mg namespace: kserve-ci-e2e-test resourceVersion: "16838" uid: d7c32f6f-cfaa-4d6f-937a-950a5560eb00 kind: Event lastTimestamp: "2026-04-24T14:39:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T14:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:45Z" name: message-dumper-raw-cf657-predictor-b44ccb865-rt4mg.18a951e83ba292ce namespace: kserve-ci-e2e-test resourceVersion: "16868" uid: 777060eb-3378-453b-8651-5d12d090b64c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-95.ec2.internal source: component: kubelet host: ip-10-0-137-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cf657-predictor-b44ccb865-rt4mg namespace: kserve-ci-e2e-test resourceVersion: "16838" uid: d7c32f6f-cfaa-4d6f-937a-950a5560eb00 kind: Event lastTimestamp: "2026-04-24T14:39:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T14:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:45Z" name: message-dumper-raw-cf657-predictor-b44ccb865-rt4mg.18a951e83c1733bf namespace: kserve-ci-e2e-test resourceVersion: "16869" uid: 4a492d34-f7b0-459d-96b4-4c219f154680 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-95.ec2.internal source: component: kubelet host: ip-10-0-137-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:41:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cf657-predictor-b44ccb865-rt4mg namespace: kserve-ci-e2e-test resourceVersion: "16838" uid: d7c32f6f-cfaa-4d6f-937a-950a5560eb00 kind: Event lastTimestamp: "2026-04-24T14:41:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T14:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:41:19Z" name: message-dumper-raw-cf657-predictor-b44ccb865-rt4mg.18a951fe1a8ce482 namespace: kserve-ci-e2e-test resourceVersion: "17707" uid: 70e2450b-f7a4-407b-a4d5-3872646bbeda reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-95.ec2.internal source: component: kubelet host: ip-10-0-137-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-cf657-predictor-b44ccb865 namespace: kserve-ci-e2e-test resourceVersion: "16810" uid: 5593575c-3fb8-47ac-81bd-498dee083a72 kind: Event lastTimestamp: "2026-04-24T14:39:44Z" message: 'Created pod: message-dumper-raw-cf657-predictor-b44ccb865-rt4mg' metadata: creationTimestamp: "2026-04-24T14:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:39:44Z" name: message-dumper-raw-cf657-predictor-b44ccb865.18a951e7e3d51cd6 namespace: kserve-ci-e2e-test resourceVersion: "16837" uid: 48ad2176-051a-4230-afa8-fa1309451bec reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-cf657-predictor namespace: kserve-ci-e2e-test resourceVersion: "16805" uid: a819d892-43bc-4aac-915c-54ec0b09c63e kind: Event lastTimestamp: "2026-04-24T14:39:44Z" message: Scaled up replica set message-dumper-raw-cf657-predictor-b44ccb865 from 0 to 1 metadata: creationTimestamp: "2026-04-24T14:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:39:44Z" name: message-dumper-raw-cf657-predictor.18a951e7e0792268 namespace: kserve-ci-e2e-test resourceVersion: "16813" uid: 2ab8ee7d-dd46-4fcd-8834-bde3cd1f25ba reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T14:39:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-cf657-predictor namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: 3e826492-63b2-4865-8e5d-5d9572435ae6 kind: Event lastTimestamp: "2026-04-24T14:40:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:40:14Z" name: message-dumper-raw-cf657-predictor.18a951eb5f0614f4 namespace: kserve-ci-e2e-test resourceVersion: "17164" uid: 070488ba-fbf5-4e8d-8201-ef733f27b519 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T14:39:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-cf657-predictor namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: 3e826492-63b2-4865-8e5d-5d9572435ae6 kind: Event lastTimestamp: "2026-04-24T14:40:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:40:14Z" name: message-dumper-raw-cf657-predictor.18a951eb5f705dc2 namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: 610dbe6a-9eda-4de9-9cb3-490e5bb34ee1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T14:39:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-cf657 namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: bf314078-cd92-4e81-8778-13f588a7cbb7 kind: Event lastTimestamp: "2026-04-24T14:39:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T14:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:39:46Z" name: message-dumper-raw-cf657.18a951e7e05500c6 namespace: kserve-ci-e2e-test resourceVersion: "16888" uid: da4b6993-d651-4d96-a165-05a94af16322 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-cf657 namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: bf314078-cd92-4e81-8778-13f588a7cbb7 kind: Event lastTimestamp: "2026-04-24T14:39:44Z" message: 'Failed to update status for InferenceService "message-dumper-raw-cf657": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-cf657": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T14:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:39:44Z" name: message-dumper-raw-cf657.18a951e7e40c1e43 namespace: kserve-ci-e2e-test resourceVersion: "16842" uid: b2ea7d5b-2b7a-4a5d-bbd3-58feb5127b64 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-cf657 namespace: kserve-ci-e2e-test resourceVersion: "16886" uid: bf314078-cd92-4e81-8778-13f588a7cbb7 kind: Event lastTimestamp: "2026-04-24T14:39:46Z" message: InferenceService [message-dumper-raw-cf657] is Ready metadata: creationTimestamp: "2026-04-24T14:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:39:46Z" name: message-dumper-raw-cf657.18a951e864ed3248 namespace: kserve-ci-e2e-test resourceVersion: "16887" uid: d1cadbce-5bc0-42ef-989a-84a436dd7a3d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T14:37:53.792707Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-a4202-598ddcf66c-jmhdd namespace: kserve-ci-e2e-test resourceVersion: "15700" uid: ceba8a83-9f4c-4aed-bac0-7b735085dba4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-a4202-598ddcf66c-jmhdd to ip-10-0-137-95.ec2.internal metadata: creationTimestamp: "2026-04-24T14:37:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T14:37:53Z" name: model-chainer-raw-a4202-598ddcf66c-jmhdd.18a951ce248d222e namespace: kserve-ci-e2e-test resourceVersion: "15708" uid: 02f5ce5d-0ced-4865-b8f2-f04187462ead reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd58b56dd-ls688 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:37:54Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-a4202-598ddcf66c-jmhdd namespace: kserve-ci-e2e-test resourceVersion: "15711" uid: ceba8a83-9f4c-4aed-bac0-7b735085dba4 kind: Event lastTimestamp: "2026-04-24T14:37:54Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T14:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T14:37:54Z" name: model-chainer-raw-a4202-598ddcf66c-jmhdd.18a951ce3d722320 namespace: kserve-ci-e2e-test resourceVersion: "15722" uid: 9052100e-8fdb-4a9c-b4e7-417772880650 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:37:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-a4202} kind: Pod name: model-chainer-raw-a4202-598ddcf66c-jmhdd namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: ceba8a83-9f4c-4aed-bac0-7b735085dba4 kind: Event lastTimestamp: "2026-04-24T14:37:54Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:fefd1106bb22731cc766906fbe03e5e23570e9f66dd8c9e94ebe1e7e3a916c30" metadata: creationTimestamp: "2026-04-24T14:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:37:54Z" name: model-chainer-raw-a4202-598ddcf66c-jmhdd.18a951ce3ead6f89 namespace: kserve-ci-e2e-test resourceVersion: "15724" uid: 1675ebe4-23b2-4a3c-8464-64e758052f3c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-95.ec2.internal source: component: kubelet host: ip-10-0-137-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:37:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-a4202} kind: Pod name: model-chainer-raw-a4202-598ddcf66c-jmhdd namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: ceba8a83-9f4c-4aed-bac0-7b735085dba4 kind: Event lastTimestamp: "2026-04-24T14:37:56Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:fefd1106bb22731cc766906fbe03e5e23570e9f66dd8c9e94ebe1e7e3a916c30" in 2.636s (2.636s including waiting). Image size: 216232505 bytes.' metadata: creationTimestamp: "2026-04-24T14:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:37:56Z" name: model-chainer-raw-a4202-598ddcf66c-jmhdd.18a951cedbd7a732 namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: 1b090825-ab97-404b-9c05-8f10f74872d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-95.ec2.internal source: component: kubelet host: ip-10-0-137-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:37:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-a4202} kind: Pod name: model-chainer-raw-a4202-598ddcf66c-jmhdd namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: ceba8a83-9f4c-4aed-bac0-7b735085dba4 kind: Event lastTimestamp: "2026-04-24T14:37:56Z" message: 'Created container: model-chainer-raw-a4202' metadata: creationTimestamp: "2026-04-24T14:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:37:56Z" name: model-chainer-raw-a4202-598ddcf66c-jmhdd.18a951cedf35bd03 namespace: kserve-ci-e2e-test resourceVersion: "15743" uid: cfc85c40-7dd3-4c6b-8fd2-384305c4a1f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-95.ec2.internal source: component: kubelet host: ip-10-0-137-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:37:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-a4202} kind: Pod name: model-chainer-raw-a4202-598ddcf66c-jmhdd namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: ceba8a83-9f4c-4aed-bac0-7b735085dba4 kind: Event lastTimestamp: "2026-04-24T14:37:56Z" message: Started container model-chainer-raw-a4202 metadata: creationTimestamp: "2026-04-24T14:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:37:56Z" name: model-chainer-raw-a4202-598ddcf66c-jmhdd.18a951cedfa5e545 namespace: kserve-ci-e2e-test resourceVersion: "15744" uid: 1926081e-3f3d-4888-909c-5f077f32dfdd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-95.ec2.internal source: component: kubelet host: ip-10-0-137-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:38:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-a4202} kind: Pod name: model-chainer-raw-a4202-598ddcf66c-jmhdd namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: ceba8a83-9f4c-4aed-bac0-7b735085dba4 kind: Event lastTimestamp: "2026-04-24T14:38:03Z" message: Stopping container model-chainer-raw-a4202 metadata: creationTimestamp: "2026-04-24T14:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:38:03Z" name: model-chainer-raw-a4202-598ddcf66c-jmhdd.18a951d07c8abc29 namespace: kserve-ci-e2e-test resourceVersion: "15824" uid: ea8b9bbd-c977-41cd-8f80-7e7143acd7b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-95.ec2.internal source: component: kubelet host: ip-10-0-137-95.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T14:38:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-a4202} kind: Pod name: model-chainer-raw-a4202-598ddcf66c-jmhdd namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: ceba8a83-9f4c-4aed-bac0-7b735085dba4 kind: Event lastTimestamp: "2026-04-24T14:38:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T14:38:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:38:33Z" name: model-chainer-raw-a4202-598ddcf66c-jmhdd.18a951d17934efe3 namespace: kserve-ci-e2e-test resourceVersion: "16185" uid: e7a91c24-87bd-4487-984c-5d07ba5baeb0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-95.ec2.internal source: component: kubelet host: ip-10-0-137-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:37:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-a4202-598ddcf66c namespace: kserve-ci-e2e-test resourceVersion: "15697" uid: 6246e31b-e4d4-45db-bf4e-e16177c525cb kind: Event lastTimestamp: "2026-04-24T14:37:53Z" message: 'Created pod: model-chainer-raw-a4202-598ddcf66c-jmhdd' metadata: creationTimestamp: "2026-04-24T14:37:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:37:53Z" name: model-chainer-raw-a4202-598ddcf66c.18a951ce23e3d20a namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: 6e3e41f1-28eb-4162-ac05-87933e57b88e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:37:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-a4202 namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: 6d7a0b9a-b738-449f-859c-d3757f7e6414 kind: Event lastTimestamp: "2026-04-24T14:37:53Z" message: Scaled up replica set model-chainer-raw-a4202-598ddcf66c from 0 to 1 metadata: creationTimestamp: "2026-04-24T14:37:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:37:53Z" name: model-chainer-raw-a4202.18a951ce22fa9055 namespace: kserve-ci-e2e-test resourceVersion: "15698" uid: b6dabe11-2bde-422e-b393-1c4fc2eae325 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:37:53Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-a4202 namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: 5fba91c7-c6ca-49eb-9919-c76f1834106b kind: Event lastTimestamp: "2026-04-24T14:37:53Z" message: InferenceGraph [model-chainer-raw-a4202] is Ready metadata: creationTimestamp: "2026-04-24T14:37:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:37:53Z" name: model-chainer-raw-a4202.18a951ce2776178e namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: 617a8dee-3ce5-4ad6-bacb-4a0be013dd2d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T14:39:34.144828Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-29d74-6d5c5cdccf-4q7cg namespace: kserve-ci-e2e-test resourceVersion: "16658" uid: ef57f6a9-2ca9-4ed1-b191-d2f663ea1d2b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-29d74-6d5c5cdccf-4q7cg to ip-10-0-137-95.ec2.internal metadata: creationTimestamp: "2026-04-24T14:39:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T14:39:34Z" name: model-chainer-raw-hpa-29d74-6d5c5cdccf-4q7cg.18a951e58200f475 namespace: kserve-ci-e2e-test resourceVersion: "16664" uid: c05d683c-3e5a-4d70-ba86-abcd6bb60a23 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd58b56dd-ls688 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:34Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-29d74-6d5c5cdccf-4q7cg namespace: kserve-ci-e2e-test resourceVersion: "16661" uid: ef57f6a9-2ca9-4ed1-b191-d2f663ea1d2b kind: Event lastTimestamp: "2026-04-24T14:39:34Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-29d74-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T14:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:34Z" name: model-chainer-raw-hpa-29d74-6d5c5cdccf-4q7cg.18a951e58c0272d5 namespace: kserve-ci-e2e-test resourceVersion: "16676" uid: 6cc5e7c1-6794-4995-803e-eae028048c8f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-95.ec2.internal source: component: kubelet host: ip-10-0-137-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:35Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-29d74-6d5c5cdccf-4q7cg namespace: kserve-ci-e2e-test resourceVersion: "16669" uid: ef57f6a9-2ca9-4ed1-b191-d2f663ea1d2b kind: Event lastTimestamp: "2026-04-24T14:39:35Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T14:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T14:39:35Z" name: model-chainer-raw-hpa-29d74-6d5c5cdccf-4q7cg.18a951e5bf044f4d namespace: kserve-ci-e2e-test resourceVersion: "16680" uid: 47360348-c02a-41f1-b2d2-784e40a8ff21 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-29d74} kind: Pod name: model-chainer-raw-hpa-29d74-6d5c5cdccf-4q7cg namespace: kserve-ci-e2e-test resourceVersion: "16661" uid: ef57f6a9-2ca9-4ed1-b191-d2f663ea1d2b kind: Event lastTimestamp: "2026-04-24T14:39:35Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:fefd1106bb22731cc766906fbe03e5e23570e9f66dd8c9e94ebe1e7e3a916c30" already present on machine metadata: creationTimestamp: "2026-04-24T14:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:35Z" name: model-chainer-raw-hpa-29d74-6d5c5cdccf-4q7cg.18a951e5c095dfcd namespace: kserve-ci-e2e-test resourceVersion: "16682" uid: 796b8431-f70c-4f53-9f3a-6212d429ea50 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-95.ec2.internal source: component: kubelet host: ip-10-0-137-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-29d74} kind: Pod name: model-chainer-raw-hpa-29d74-6d5c5cdccf-4q7cg namespace: kserve-ci-e2e-test resourceVersion: "16661" uid: ef57f6a9-2ca9-4ed1-b191-d2f663ea1d2b kind: Event lastTimestamp: "2026-04-24T14:39:35Z" message: 'Created container: model-chainer-raw-hpa-29d74' metadata: creationTimestamp: "2026-04-24T14:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:35Z" name: model-chainer-raw-hpa-29d74-6d5c5cdccf-4q7cg.18a951e5c36bb570 namespace: kserve-ci-e2e-test resourceVersion: "16683" uid: f483c964-88d0-47ad-bfa9-33f2d669f99b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-95.ec2.internal source: component: kubelet host: ip-10-0-137-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-29d74} kind: Pod name: model-chainer-raw-hpa-29d74-6d5c5cdccf-4q7cg namespace: kserve-ci-e2e-test resourceVersion: "16661" uid: ef57f6a9-2ca9-4ed1-b191-d2f663ea1d2b kind: Event lastTimestamp: "2026-04-24T14:39:35Z" message: Started container model-chainer-raw-hpa-29d74 metadata: creationTimestamp: "2026-04-24T14:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:35Z" name: model-chainer-raw-hpa-29d74-6d5c5cdccf-4q7cg.18a951e5c3e376ec namespace: kserve-ci-e2e-test resourceVersion: "16684" uid: e290388b-d6ca-4dc4-8708-4ac03d3847a5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-95.ec2.internal source: component: kubelet host: ip-10-0-137-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-29d74} kind: Pod name: model-chainer-raw-hpa-29d74-6d5c5cdccf-4q7cg namespace: kserve-ci-e2e-test resourceVersion: "16661" uid: ef57f6a9-2ca9-4ed1-b191-d2f663ea1d2b kind: Event lastTimestamp: "2026-04-24T14:39:44Z" message: Stopping container model-chainer-raw-hpa-29d74 metadata: creationTimestamp: "2026-04-24T14:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:39:44Z" name: model-chainer-raw-hpa-29d74-6d5c5cdccf-4q7cg.18a951e7d952642d namespace: kserve-ci-e2e-test resourceVersion: "16784" uid: c6c19f21-3bca-49ac-be7a-8fa2fcfe0691 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-95.ec2.internal source: component: kubelet host: ip-10-0-137-95.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T14:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-29d74} kind: Pod name: model-chainer-raw-hpa-29d74-6d5c5cdccf-4q7cg namespace: kserve-ci-e2e-test resourceVersion: "16661" uid: ef57f6a9-2ca9-4ed1-b191-d2f663ea1d2b kind: Event lastTimestamp: "2026-04-24T14:40:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T14:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:40:11Z" name: model-chainer-raw-hpa-29d74-6d5c5cdccf-4q7cg.18a951e85dde31a8 namespace: kserve-ci-e2e-test resourceVersion: "17138" uid: 1fecc911-002f-4d0f-8bb8-9705b46ff2d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-95.ec2.internal source: component: kubelet host: ip-10-0-137-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-29d74-6d5c5cdccf namespace: kserve-ci-e2e-test resourceVersion: "16654" uid: a8bf3a05-2737-43a3-9546-649f2405605b kind: Event lastTimestamp: "2026-04-24T14:39:34Z" message: 'Created pod: model-chainer-raw-hpa-29d74-6d5c5cdccf-4q7cg' metadata: creationTimestamp: "2026-04-24T14:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:39:34Z" name: model-chainer-raw-hpa-29d74-6d5c5cdccf.18a951e581899cd2 namespace: kserve-ci-e2e-test resourceVersion: "16660" uid: dc0076d7-5274-4e07-b92e-5a2aa94a1a86 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-29d74 namespace: kserve-ci-e2e-test resourceVersion: "16652" uid: 6b546b6e-6846-46c6-a421-dd6300a7120b kind: Event lastTimestamp: "2026-04-24T14:39:34Z" message: Scaled up replica set model-chainer-raw-hpa-29d74-6d5c5cdccf from 0 to 1 metadata: creationTimestamp: "2026-04-24T14:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:39:34Z" name: model-chainer-raw-hpa-29d74.18a951e5804ba857 namespace: kserve-ci-e2e-test resourceVersion: "16656" uid: 5eddecfe-b8c3-4514-85e3-4b33ce8198fd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:39:34Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-29d74 namespace: kserve-ci-e2e-test resourceVersion: "16671" uid: 61cf15d8-e8f1-4f2e-9044-c943af29b8d3 kind: Event lastTimestamp: "2026-04-24T14:39:34Z" message: InferenceGraph [model-chainer-raw-hpa-29d74] is Ready metadata: creationTimestamp: "2026-04-24T14:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:39:34Z" name: model-chainer-raw-hpa-29d74.18a951e5845d89eb namespace: kserve-ci-e2e-test resourceVersion: "16673" uid: f14e6878-11db-4b06-b8d3-e9ec719a8ac5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T14:45:34.485835Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6 namespace: kserve-ci-e2e-test resourceVersion: "19840" uid: a5e8090e-8f94-420c-bb45-52d442cf00d0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-de5d9-predictor-5945d675c9-qn4w6 to ip-10-0-143-92.ec2.internal metadata: creationTimestamp: "2026-04-24T14:45:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T14:45:34Z" name: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6.18a9523968006b6c namespace: kserve-ci-e2e-test resourceVersion: "19849" uid: adc33c58-4a4e-4b50-894e-dc3516242b73 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd58b56dd-ls688 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:34Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6 namespace: kserve-ci-e2e-test resourceVersion: "19853" uid: a5e8090e-8f94-420c-bb45-52d442cf00d0 kind: Event lastTimestamp: "2026-04-24T14:45:34Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T14:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T14:45:34Z" name: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6.18a9523980e10d7e namespace: kserve-ci-e2e-test resourceVersion: "19860" uid: d83cc6c9-b8e8-4be2-a9ad-c792cd7ca958 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6 namespace: kserve-ci-e2e-test resourceVersion: "19846" uid: a5e8090e-8f94-420c-bb45-52d442cf00d0 kind: Event lastTimestamp: "2026-04-24T14:45:34Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3370fbcd366f44a8710ac5ab5a02712bec0322005baa64fee46cbff50aae7c18" already present on machine metadata: creationTimestamp: "2026-04-24T14:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:45:34Z" name: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6.18a9523982226604 namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: 2b093647-aaeb-4ab9-b770-67d092120189 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6 namespace: kserve-ci-e2e-test resourceVersion: "19846" uid: a5e8090e-8f94-420c-bb45-52d442cf00d0 kind: Event lastTimestamp: "2026-04-24T14:45:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T14:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:45:34Z" name: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6.18a95239851dc77b namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: 6f982740-76bf-4f69-b446-9a8250683771 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6 namespace: kserve-ci-e2e-test resourceVersion: "19846" uid: a5e8090e-8f94-420c-bb45-52d442cf00d0 kind: Event lastTimestamp: "2026-04-24T14:45:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T14:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:45:34Z" name: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6.18a95239858f7d8a namespace: kserve-ci-e2e-test resourceVersion: "19866" uid: 5dada2f0-a37c-4102-a9a2-074c977372ad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6 namespace: kserve-ci-e2e-test resourceVersion: "19846" uid: a5e8090e-8f94-420c-bb45-52d442cf00d0 kind: Event lastTimestamp: "2026-04-24T14:45:38Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-04-24T14:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:45:38Z" name: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6.18a9523a736211ef namespace: kserve-ci-e2e-test resourceVersion: "19903" uid: 822fe5bd-a90b-43c4-9dd1-31e75e58980a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6 namespace: kserve-ci-e2e-test resourceVersion: "19846" uid: a5e8090e-8f94-420c-bb45-52d442cf00d0 kind: Event lastTimestamp: "2026-04-24T14:45:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T14:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:45:39Z" name: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6.18a9523a7631de84 namespace: kserve-ci-e2e-test resourceVersion: "19907" uid: badb0f5b-c76a-408a-8cf0-3c491d0140ca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6 namespace: kserve-ci-e2e-test resourceVersion: "19846" uid: a5e8090e-8f94-420c-bb45-52d442cf00d0 kind: Event lastTimestamp: "2026-04-24T14:45:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T14:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:45:39Z" name: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6.18a9523a76a5c90b namespace: kserve-ci-e2e-test resourceVersion: "19908" uid: 2ef484cc-411f-439a-93b9-d41fa6fc5b25 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T14:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6 namespace: kserve-ci-e2e-test resourceVersion: "19846" uid: a5e8090e-8f94-420c-bb45-52d442cf00d0 kind: Event lastTimestamp: "2026-04-24T14:46:55Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T14:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:46:55Z" name: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6.18a9523aaf4d417c namespace: kserve-ci-e2e-test resourceVersion: "20517" uid: 81db8077-7ffe-4be3-b9d1-1768f7238541 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:46:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6 namespace: kserve-ci-e2e-test resourceVersion: "19846" uid: a5e8090e-8f94-420c-bb45-52d442cf00d0 kind: Event lastTimestamp: "2026-04-24T14:46:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T14:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:46:54Z" name: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6.18a9524c0f77a418 namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: 26646032-5922-4b28-847c-48d070722273 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-de5d9-predictor-5945d675c9 namespace: kserve-ci-e2e-test resourceVersion: "19830" uid: 02326133-7334-4115-8a1e-3f1f98f55e4f kind: Event lastTimestamp: "2026-04-24T14:45:34Z" message: 'Created pod: raw-sklearn-de5d9-predictor-5945d675c9-qn4w6' metadata: creationTimestamp: "2026-04-24T14:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:45:34Z" name: raw-sklearn-de5d9-predictor-5945d675c9.18a9523967764ae7 namespace: kserve-ci-e2e-test resourceVersion: "19845" uid: 8d3e98f6-b06d-4c3d-a8e1-7079b7efa0ba reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-de5d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19829" uid: 15dfad1d-e3dc-421c-9ac8-cf8507151be1 kind: Event lastTimestamp: "2026-04-24T14:45:34Z" message: Scaled up replica set raw-sklearn-de5d9-predictor-5945d675c9 from 0 to 1 metadata: creationTimestamp: "2026-04-24T14:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:45:34Z" name: raw-sklearn-de5d9-predictor.18a95239649c18d0 namespace: kserve-ci-e2e-test resourceVersion: "19832" uid: d870b122-875d-44de-8248-b67456c4f7b1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-de5d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19831" uid: 86269c3f-2b6d-442a-a012-2681fdd6971d kind: Event lastTimestamp: "2026-04-24T14:45:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:45:49Z" name: raw-sklearn-de5d9-predictor.18a9523ce380a36d namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: adba894e-3dcb-412b-929a-1cc69eda4a97 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-de5d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19831" uid: 86269c3f-2b6d-442a-a012-2681fdd6971d kind: Event lastTimestamp: "2026-04-24T14:45:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:45:49Z" name: raw-sklearn-de5d9-predictor.18a9523ce3e56744 namespace: kserve-ci-e2e-test resourceVersion: "19991" uid: 53bf6e5b-ed05-44d2-bed1-b587d7cffefd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T14:46:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-de5d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19990" uid: 86269c3f-2b6d-442a-a012-2681fdd6971d kind: Event lastTimestamp: "2026-04-24T14:46:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T14:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:46:49Z" name: raw-sklearn-de5d9-predictor.18a95240630b9a2f namespace: kserve-ci-e2e-test resourceVersion: "20414" uid: ddb5fe1a-db2f-4036-b807-c0187a782587 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T14:46:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-de5d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19990" uid: 86269c3f-2b6d-442a-a012-2681fdd6971d kind: Event lastTimestamp: "2026-04-24T14:46:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T14:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:46:49Z" name: raw-sklearn-de5d9-predictor.18a95240637b25b3 namespace: kserve-ci-e2e-test resourceVersion: "20415" uid: 5f1b1128-2b25-4ac5-88b1-2675917d958d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-24T14:45:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-de5d9 namespace: kserve-ci-e2e-test resourceVersion: "19822" uid: c034509d-3de8-4c40-81da-e724258234d8 kind: Event lastTimestamp: "2026-04-24T14:46:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T14:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:46:45Z" name: raw-sklearn-de5d9.18a9523964cc6de0 namespace: kserve-ci-e2e-test resourceVersion: "20388" uid: 8a8f9918-a908-46cc-80b6-4c554a50eee1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:45:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-de5d9 namespace: kserve-ci-e2e-test resourceVersion: "19822" uid: c034509d-3de8-4c40-81da-e724258234d8 kind: Event lastTimestamp: "2026-04-24T14:45:34Z" message: 'Failed to update status for InferenceService "raw-sklearn-de5d9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-de5d9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T14:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:45:34Z" name: raw-sklearn-de5d9.18a9523965592f29 namespace: kserve-ci-e2e-test resourceVersion: "19837" uid: 1592e333-e747-4ea8-9b79-fb045412a743 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:46:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-de5d9 namespace: kserve-ci-e2e-test resourceVersion: "20386" uid: c034509d-3de8-4c40-81da-e724258234d8 kind: Event lastTimestamp: "2026-04-24T14:46:45Z" message: InferenceService [raw-sklearn-de5d9] is Ready metadata: creationTimestamp: "2026-04-24T14:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:46:45Z" name: raw-sklearn-de5d9.18a95249e14e6078 namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: 6ff09f11-ecb3-430f-aefb-9dda1bd05353 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T14:46:54.672320Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658 namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: 1d665915-0b4d-4cf2-b448-dba8cdcbd553 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658 to ip-10-0-143-92.ec2.internal metadata: creationTimestamp: "2026-04-24T14:46:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T14:46:54Z" name: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658.18a9524c137d1bfd namespace: kserve-ci-e2e-test resourceVersion: "20495" uid: 93527cb9-9657-4483-8ff5-825e3d9d2d42 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd58b56dd-ls688 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:46:55Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658 namespace: kserve-ci-e2e-test resourceVersion: "20498" uid: 1d665915-0b4d-4cf2-b448-dba8cdcbd553 kind: Event lastTimestamp: "2026-04-24T14:46:55Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T14:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T14:46:55Z" name: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658.18a9524c2c770ce7 namespace: kserve-ci-e2e-test resourceVersion: "20513" uid: 54cd1fd6-7df7-470d-8b6e-d6f3b6fe744f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:46:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658 namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: 1d665915-0b4d-4cf2-b448-dba8cdcbd553 kind: Event lastTimestamp: "2026-04-24T14:46:55Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3370fbcd366f44a8710ac5ab5a02712bec0322005baa64fee46cbff50aae7c18" already present on machine metadata: creationTimestamp: "2026-04-24T14:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:46:55Z" name: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658.18a9524c2dcc821c namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: e65b8726-f519-4b55-8f92-ea1f33cb18a6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:46:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658 namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: 1d665915-0b4d-4cf2-b448-dba8cdcbd553 kind: Event lastTimestamp: "2026-04-24T14:46:55Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T14:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:46:55Z" name: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658.18a9524c30e72b83 namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: 493e3c29-e62e-481b-b54e-2b5ddd585b59 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:46:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658 namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: 1d665915-0b4d-4cf2-b448-dba8cdcbd553 kind: Event lastTimestamp: "2026-04-24T14:46:55Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T14:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:46:55Z" name: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658.18a9524c316508b2 namespace: kserve-ci-e2e-test resourceVersion: "20519" uid: ed436e43-fbd5-4f0f-8bf3-04ce29e8e99c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:46:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658 namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: 1d665915-0b4d-4cf2-b448-dba8cdcbd553 kind: Event lastTimestamp: "2026-04-24T14:46:59Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-04-24T14:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:46:59Z" name: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658.18a9524d21a3ad13 namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: 50e2d36b-febf-42fc-899e-94acbac24adb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:46:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658 namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: 1d665915-0b4d-4cf2-b448-dba8cdcbd553 kind: Event lastTimestamp: "2026-04-24T14:46:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T14:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:46:59Z" name: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658.18a9524d24f79060 namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: 2961d259-16df-47d2-bf50-e14a88922e02 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:46:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658 namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: 1d665915-0b4d-4cf2-b448-dba8cdcbd553 kind: Event lastTimestamp: "2026-04-24T14:46:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T14:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:46:59Z" name: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658.18a9524d2575e12b namespace: kserve-ci-e2e-test resourceVersion: "20559" uid: 68313dd0-6d6e-49a5-a301-4c18cb57be15 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T14:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658 namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: 1d665915-0b4d-4cf2-b448-dba8cdcbd553 kind: Event lastTimestamp: "2026-04-24T14:48:01Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T14:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:48:01Z" name: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658.18a9524d5da774fa namespace: kserve-ci-e2e-test resourceVersion: "21008" uid: f991f533-b3a3-4f65-8781-b5ded4c01733 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658 namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: 1d665915-0b4d-4cf2-b448-dba8cdcbd553 kind: Event lastTimestamp: "2026-04-24T14:48:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T14:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T14:48:14Z" name: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658.18a9525ebb766e2b namespace: kserve-ci-e2e-test resourceVersion: "21128" uid: 49645ebb-f6db-4397-9fb4-3bb69e56b974 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-92.ec2.internal source: component: kubelet host: ip-10-0-143-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:46:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-a818b-predictor-5fcf798447 namespace: kserve-ci-e2e-test resourceVersion: "20479" uid: 06618939-b805-4655-9848-9e4298252daf kind: Event lastTimestamp: "2026-04-24T14:46:54Z" message: 'Created pod: raw-sklearn-runtime-a818b-predictor-5fcf798447-sj658' metadata: creationTimestamp: "2026-04-24T14:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:46:54Z" name: raw-sklearn-runtime-a818b-predictor-5fcf798447.18a9524c130786cd namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: b9dffb92-dd13-4079-b494-bc5081fb6286 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:46:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-a818b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20477" uid: d79793cd-b285-4aac-a28b-624e3a995c5c kind: Event lastTimestamp: "2026-04-24T14:46:54Z" message: Scaled up replica set raw-sklearn-runtime-a818b-predictor-5fcf798447 from 0 to 1 metadata: creationTimestamp: "2026-04-24T14:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:46:54Z" name: raw-sklearn-runtime-a818b-predictor.18a9524c10025a72 namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: 6aa69f67-f9fe-4870-b3c9-9a6f5bf121c5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T14:47:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-a818b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20480" uid: b2b9d491-eb54-4260-a99e-915dd5aa9e85 kind: Event lastTimestamp: "2026-04-24T14:47:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:47:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:47:24Z" name: raw-sklearn-runtime-a818b-predictor.18a9524f8f0c7087 namespace: kserve-ci-e2e-test resourceVersion: "20749" uid: da07822e-5c50-4afb-a83d-1c9f5464ec89 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T14:47:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-a818b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20480" uid: b2b9d491-eb54-4260-a99e-915dd5aa9e85 kind: Event lastTimestamp: "2026-04-24T14:47:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T14:47:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:47:24Z" name: raw-sklearn-runtime-a818b-predictor.18a9524f8f717485 namespace: kserve-ci-e2e-test resourceVersion: "20750" uid: afbd2fb8-d284-467b-a2ed-38cc5e7d1600 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T14:47:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-a818b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: b2b9d491-eb54-4260-a99e-915dd5aa9e85 kind: Event lastTimestamp: "2026-04-24T14:48:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T14:47:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:48:09Z" name: raw-sklearn-runtime-a818b-predictor.18a952568d6de40c namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: 4fc79a11-73a2-405b-b2b2-fb90ff08b916 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T14:47:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-a818b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: b2b9d491-eb54-4260-a99e-915dd5aa9e85 kind: Event lastTimestamp: "2026-04-24T14:48:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T14:47:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T14:48:09Z" name: raw-sklearn-runtime-a818b-predictor.18a952568dd397c9 namespace: kserve-ci-e2e-test resourceVersion: "21066" uid: b41ac392-805c-4d9a-8446-fd391178417f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-24T14:46:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-a818b namespace: kserve-ci-e2e-test resourceVersion: "20453" uid: b1351e4c-d902-4223-a2b9-ab0b78757410 kind: Event lastTimestamp: "2026-04-24T14:48:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T14:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:48:11Z" name: raw-sklearn-runtime-a818b.18a9524c100c8e98 namespace: kserve-ci-e2e-test resourceVersion: "21085" uid: 06aeec11-5f61-43ad-9acd-c0b24afb9148 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T14:46:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-a818b namespace: kserve-ci-e2e-test resourceVersion: "20453" uid: b1351e4c-d902-4223-a2b9-ab0b78757410 kind: Event lastTimestamp: "2026-04-24T14:46:54Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-a818b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-a818b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T14:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:46:54Z" name: raw-sklearn-runtime-a818b.18a9524c1094e001 namespace: kserve-ci-e2e-test resourceVersion: "20501" uid: b4ebb1e6-80fd-4b03-972b-aa5a0e66a361 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:46:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-a818b namespace: kserve-ci-e2e-test resourceVersion: "20466" uid: b1351e4c-d902-4223-a2b9-ab0b78757410 kind: Event lastTimestamp: "2026-04-24T14:46:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-a818b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T14:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:46:54Z" name: raw-sklearn-runtime-a818b.18a9524c1561be4e namespace: kserve-ci-e2e-test resourceVersion: "20503" uid: 44a752e5-cf7e-47e2-85aa-d47b7c4be1d9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T14:48:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-a818b namespace: kserve-ci-e2e-test resourceVersion: "21082" uid: b1351e4c-d902-4223-a2b9-ab0b78757410 kind: Event lastTimestamp: "2026-04-24T14:48:11Z" message: InferenceService [raw-sklearn-runtime-a818b] is Ready metadata: creationTimestamp: "2026-04-24T14:48:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T14:48:11Z" name: raw-sklearn-runtime-a818b.18a9525dea0f982c namespace: kserve-ci-e2e-test resourceVersion: "21084" uid: 49cb2544-4741-46f2-8bbf-a944d95ba5cb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "21321"