--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:19:03.514157Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-9f5836-predictor-585f8bcf78-z24bn namespace: kserve-ci-e2e-test resourceVersion: "19787" uid: ea895d81-0422-4054-9fb6-b364dfc42f26 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-9f5836-predictor-585f8bcf78-z24bn to ip-10-0-141-131.ec2.internal metadata: creationTimestamp: "2026-04-16T14:19:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:19:03Z" name: isvc-init-fail-9f5836-predictor-585f8bcf78-z24bn.18a6dc22703b3fba namespace: kserve-ci-e2e-test resourceVersion: "19791" uid: 3ac27ee7-7ded-4d25-8aa8-272193ae6051 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-66bdb995df-m8nk5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:03Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-9f5836-predictor-585f8bcf78-z24bn namespace: kserve-ci-e2e-test resourceVersion: "19796" uid: ea895d81-0422-4054-9fb6-b364dfc42f26 kind: Event lastTimestamp: "2026-04-16T14:19:03Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:19:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:19:03Z" name: isvc-init-fail-9f5836-predictor-585f8bcf78-z24bn.18a6dc2289951d84 namespace: kserve-ci-e2e-test resourceVersion: "19809" uid: caa5016a-7c98-4a97-90ad-1bb105fec6b3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:19:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-9f5836-predictor-585f8bcf78-z24bn namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: ea895d81-0422-4054-9fb6-b364dfc42f26 kind: Event lastTimestamp: "2026-04-16T14:19:08Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:16546eaa530f8a9a22c60a7ffa82a496f33cd9dfcf4989c8baef0968e8392589" already present on machine metadata: creationTimestamp: "2026-04-16T14:19:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:08Z" name: isvc-init-fail-9f5836-predictor-585f8bcf78-z24bn.18a6dc228adefe84 namespace: kserve-ci-e2e-test resourceVersion: "19853" uid: d456ce7a-daf0-4b11-9a5a-855a84707ff2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:19:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-9f5836-predictor-585f8bcf78-z24bn namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: ea895d81-0422-4054-9fb6-b364dfc42f26 kind: Event lastTimestamp: "2026-04-16T14:19:08Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:19:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:08Z" name: isvc-init-fail-9f5836-predictor-585f8bcf78-z24bn.18a6dc228e097e48 namespace: kserve-ci-e2e-test resourceVersion: "19857" uid: e25f2f63-5430-4dd3-8261-f269a79a34dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:19:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-9f5836-predictor-585f8bcf78-z24bn namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: ea895d81-0422-4054-9fb6-b364dfc42f26 kind: Event lastTimestamp: "2026-04-16T14:19:08Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:19:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:08Z" name: isvc-init-fail-9f5836-predictor-585f8bcf78-z24bn.18a6dc228e6b0175 namespace: kserve-ci-e2e-test resourceVersion: "19860" uid: 4c1f2eb3-9be7-45ca-86b8-78b506e6f431 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-9f5836-predictor-585f8bcf78-z24bn namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: ea895d81-0422-4054-9fb6-b364dfc42f26 kind: Event lastTimestamp: "2026-04-16T14:19:13Z" message: Back-off restarting failed container storage-initializer in pod isvc-init-fail-9f5836-predictor-585f8bcf78-z24bn_kserve-ci-e2e-test(ea895d81-0422-4054-9fb6-b364dfc42f26) metadata: creationTimestamp: "2026-04-16T14:19:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:13Z" name: isvc-init-fail-9f5836-predictor-585f8bcf78-z24bn.18a6dc24bfad148f namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: 6183ef6c-c000-4b52-a24f-2c75fa32ce1c reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-9f5836-predictor-585f8bcf78 namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: 9b791973-7656-430d-8774-8d3ca60ebf34 kind: Event lastTimestamp: "2026-04-16T14:19:03Z" message: 'Created pod: isvc-init-fail-9f5836-predictor-585f8bcf78-z24bn' metadata: creationTimestamp: "2026-04-16T14:19:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:19:03Z" name: isvc-init-fail-9f5836-predictor-585f8bcf78.18a6dc226fcfde6a namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: effdd5bc-0951-4b91-bba2-06db4f6ad756 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-9f5836-predictor namespace: kserve-ci-e2e-test resourceVersion: "19773" uid: e58d7890-1f40-44e6-adf1-f6d6a6a2f1c3 kind: Event lastTimestamp: "2026-04-16T14:19:03Z" message: Scaled up replica set isvc-init-fail-9f5836-predictor-585f8bcf78 from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:19:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:19:03Z" name: isvc-init-fail-9f5836-predictor.18a6dc226d0e94d6 namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 87c6d4cd-02ef-4be3-8220-0e86d664bc57 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-16T14:19:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-9f5836 namespace: kserve-ci-e2e-test resourceVersion: "19751" uid: f0240d63-9e2a-414a-ae5a-1d021596a905 kind: Event lastTimestamp: "2026-04-16T14:19:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:19:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:19:08Z" name: isvc-init-fail-9f5836.18a6dc226d16d7e1 namespace: kserve-ci-e2e-test resourceVersion: "19861" uid: 6ef34304-5c04-4fe6-adef-09850ff115e6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:19:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-9f5836 namespace: kserve-ci-e2e-test resourceVersion: "19751" uid: f0240d63-9e2a-414a-ae5a-1d021596a905 kind: Event lastTimestamp: "2026-04-16T14:19:03Z" message: 'Failed to update status for InferenceService "isvc-init-fail-9f5836": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-9f5836": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:19:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:19:03Z" name: isvc-init-fail-9f5836.18a6dc226d963c36 namespace: kserve-ci-e2e-test resourceVersion: "19797" uid: 7ee5b70b-fed1-4954-a6ec-6f61751ff9bf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:19:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-9f5836 namespace: kserve-ci-e2e-test resourceVersion: "19751" uid: f0240d63-9e2a-414a-ae5a-1d021596a905 kind: Event lastTimestamp: "2026-04-16T14:19:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-9f5836": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:19:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:19:03Z" name: isvc-init-fail-9f5836.18a6dc226d967e01 namespace: kserve-ci-e2e-test resourceVersion: "19798" uid: 38c35b45-0b80-40b4-a9d7-1857f50f0b01 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:13:31.653213Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg namespace: kserve-ci-e2e-test resourceVersion: "17057" uid: 710cab95-3f1a-48b7-b29b-1c6b14b8887e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg to ip-10-0-141-131.ec2.internal metadata: creationTimestamp: "2026-04-16T14:13:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:13:31Z" name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg.18a6dbd52bc713a7 namespace: kserve-ci-e2e-test resourceVersion: "17061" uid: f613c549-6ccb-4230-875b-e0316a78adc1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-66bdb995df-m8nk5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:32Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg namespace: kserve-ci-e2e-test resourceVersion: "17066" uid: 710cab95-3f1a-48b7-b29b-1c6b14b8887e kind: Event lastTimestamp: "2026-04-16T14:13:32Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:13:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:13:32Z" name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg.18a6dbd544ee1b2e namespace: kserve-ci-e2e-test resourceVersion: "17074" uid: bf38883a-1826-456b-9694-af4617a11493 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg namespace: kserve-ci-e2e-test resourceVersion: "17059" uid: 710cab95-3f1a-48b7-b29b-1c6b14b8887e kind: Event lastTimestamp: "2026-04-16T14:13:32Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:16546eaa530f8a9a22c60a7ffa82a496f33cd9dfcf4989c8baef0968e8392589" already present on machine metadata: creationTimestamp: "2026-04-16T14:13:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:32Z" name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg.18a6dbd54634df0a namespace: kserve-ci-e2e-test resourceVersion: "17076" uid: ecf6f40f-3cd9-4edc-9f3a-25e46ddac751 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg namespace: kserve-ci-e2e-test resourceVersion: "17059" uid: 710cab95-3f1a-48b7-b29b-1c6b14b8887e kind: Event lastTimestamp: "2026-04-16T14:13:32Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:13:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:32Z" name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg.18a6dbd5496b7874 namespace: kserve-ci-e2e-test resourceVersion: "17078" uid: a18d0846-9b09-4f26-b26e-e557708dbca5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg namespace: kserve-ci-e2e-test resourceVersion: "17059" uid: 710cab95-3f1a-48b7-b29b-1c6b14b8887e kind: Event lastTimestamp: "2026-04-16T14:13:32Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:13:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:32Z" name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg.18a6dbd549db8bdb namespace: kserve-ci-e2e-test resourceVersion: "17079" uid: 0b4136f1-b644-4c67-abeb-9946ef6fbb25 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg namespace: kserve-ci-e2e-test resourceVersion: "17059" uid: 710cab95-3f1a-48b7-b29b-1c6b14b8887e kind: Event lastTimestamp: "2026-04-16T14:13:36Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1362" already present on machine metadata: creationTimestamp: "2026-04-16T14:13:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:36Z" name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg.18a6dbd645ca89d9 namespace: kserve-ci-e2e-test resourceVersion: "17115" uid: d46456f2-6d2b-4bfd-b667-385fe349f471 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg namespace: kserve-ci-e2e-test resourceVersion: "17059" uid: 710cab95-3f1a-48b7-b29b-1c6b14b8887e kind: Event lastTimestamp: "2026-04-16T14:13:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:13:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:36Z" name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg.18a6dbd64960cf44 namespace: kserve-ci-e2e-test resourceVersion: "17120" uid: a4f54762-5536-46fc-b8a2-08ceb6f14c7d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg namespace: kserve-ci-e2e-test resourceVersion: "17059" uid: 710cab95-3f1a-48b7-b29b-1c6b14b8887e kind: Event lastTimestamp: "2026-04-16T14:13:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:13:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:36Z" name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg.18a6dbd649d59eaf namespace: kserve-ci-e2e-test resourceVersion: "17121" uid: 56c0395d-ec17-42bc-8712-5e4c082778a2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg namespace: kserve-ci-e2e-test resourceVersion: "17059" uid: 710cab95-3f1a-48b7-b29b-1c6b14b8887e kind: Event lastTimestamp: "2026-04-16T14:13:36Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:cda273db4b40777305d78f4b7f8bde558e921e8f875775edae595cda2e7cfa4b" already present on machine metadata: creationTimestamp: "2026-04-16T14:13:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:36Z" name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg.18a6dbd649e2457b namespace: kserve-ci-e2e-test resourceVersion: "17123" uid: fa495a80-5133-42f1-a8b1-06becf693338 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg namespace: kserve-ci-e2e-test resourceVersion: "17059" uid: 710cab95-3f1a-48b7-b29b-1c6b14b8887e kind: Event lastTimestamp: "2026-04-16T14:13:36Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-16T14:13:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:36Z" name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg.18a6dbd64d6436f5 namespace: kserve-ci-e2e-test resourceVersion: "17124" uid: ad804123-fa5a-4fb2-b5ff-e47ebf11416e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg namespace: kserve-ci-e2e-test resourceVersion: "17059" uid: 710cab95-3f1a-48b7-b29b-1c6b14b8887e kind: Event lastTimestamp: "2026-04-16T14:13:36Z" message: Started container agent metadata: creationTimestamp: "2026-04-16T14:13:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:36Z" name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg.18a6dbd64de997b3 namespace: kserve-ci-e2e-test resourceVersion: "17125" uid: 7dba48e0-cafb-48eb-b7fa-b84177b7168f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T14:13:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg namespace: kserve-ci-e2e-test resourceVersion: "17059" uid: 710cab95-3f1a-48b7-b29b-1c6b14b8887e kind: Event lastTimestamp: "2026-04-16T14:15:19Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T14:13:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:15:19Z" name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg.18a6dbd681bdef5e namespace: kserve-ci-e2e-test resourceVersion: "17976" uid: a3546a80-f4fb-4341-99ae-1f2a7a484520 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-16T14:13:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg namespace: kserve-ci-e2e-test resourceVersion: "17059" uid: 710cab95-3f1a-48b7-b29b-1c6b14b8887e kind: Event lastTimestamp: "2026-04-16T14:15:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T14:13:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:15:19Z" name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg.18a6dbd6bda681dc namespace: kserve-ci-e2e-test resourceVersion: "17977" uid: c480e237-bbef-4996-bcc5-e2eec9996d30 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg namespace: kserve-ci-e2e-test resourceVersion: "17059" uid: 710cab95-3f1a-48b7-b29b-1c6b14b8887e kind: Event lastTimestamp: "2026-04-16T14:14:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:14:56Z" name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg.18a6dbe903ba9760 namespace: kserve-ci-e2e-test resourceVersion: "17754" uid: 9cbd7713-2dba-4bbc-9702-7cc341a90153 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg namespace: kserve-ci-e2e-test resourceVersion: "17059" uid: 710cab95-3f1a-48b7-b29b-1c6b14b8887e kind: Event lastTimestamp: "2026-04-16T14:14:56Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-16T14:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:14:56Z" name: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg.18a6dbe903bc846c namespace: kserve-ci-e2e-test resourceVersion: "17755" uid: dc0e4f73-8d08-4470-a916-90052dda768e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-7cc0c-predictor-7b494555d9 namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: 1d6903a6-81b1-4d55-848e-9b6b9f0f89df kind: Event lastTimestamp: "2026-04-16T14:13:31Z" message: 'Created pod: isvc-logger-raw-7cc0c-predictor-7b494555d9-95ltg' metadata: creationTimestamp: "2026-04-16T14:13:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:31Z" name: isvc-logger-raw-7cc0c-predictor-7b494555d9.18a6dbd52b60032b namespace: kserve-ci-e2e-test resourceVersion: "17058" uid: 2e7437aa-63bc-4e25-963f-16c5698d0f74 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-7cc0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17044" uid: 0a669a55-6c48-4460-8244-a8337143f036 kind: Event lastTimestamp: "2026-04-16T14:13:31Z" message: Scaled up replica set isvc-logger-raw-7cc0c-predictor-7b494555d9 from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:13:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:31Z" name: isvc-logger-raw-7cc0c-predictor.18a6dbd5284cbde2 namespace: kserve-ci-e2e-test resourceVersion: "17049" uid: 2849d717-3006-4424-b3e5-6a26a66da5ad reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-7cc0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17045" uid: 74841bfe-ef81-4eeb-8aa8-de7522660668 kind: Event lastTimestamp: "2026-04-16T14:13:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:46Z" name: isvc-logger-raw-7cc0c-predictor.18a6dbd8a800b167 namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: e9105815-79fc-4563-9bd6-b21bf9dd9f4c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-7cc0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17045" uid: 74841bfe-ef81-4eeb-8aa8-de7522660668 kind: Event lastTimestamp: "2026-04-16T14:13:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:46Z" name: isvc-logger-raw-7cc0c-predictor.18a6dbd8a90becd9 namespace: kserve-ci-e2e-test resourceVersion: "17207" uid: a9b9c52a-2a63-4b86-a151-75e2c703866f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T14:14:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-7cc0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17206" uid: 74841bfe-ef81-4eeb-8aa8-de7522660668 kind: Event lastTimestamp: "2026-04-16T14:14:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:14:46Z" name: isvc-logger-raw-7cc0c-predictor.18a6dbdc27f7141d namespace: kserve-ci-e2e-test resourceVersion: "17639" uid: 5c76a6bb-d7fd-4eed-a064-18d957226c54 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T14:14:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-7cc0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17206" uid: 74841bfe-ef81-4eeb-8aa8-de7522660668 kind: Event lastTimestamp: "2026-04-16T14:14:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:14:46Z" name: isvc-logger-raw-7cc0c-predictor.18a6dbdc285e4019 namespace: kserve-ci-e2e-test resourceVersion: "17640" uid: adbf9c80-2f5a-423b-9ded-f42fd9a6b1ba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T14:13:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-7cc0c namespace: kserve-ci-e2e-test resourceVersion: "17036" uid: 4abcbdd1-700e-42b5-8b85-7a603794176b kind: Event lastTimestamp: "2026-04-16T14:14:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:13:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:14:49Z" name: isvc-logger-raw-7cc0c.18a6dbd52829cdc7 namespace: kserve-ci-e2e-test resourceVersion: "17667" uid: 1b0d8ce0-5a87-46ee-b057-724d16e1ab87 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-7cc0c namespace: kserve-ci-e2e-test resourceVersion: "17036" uid: 4abcbdd1-700e-42b5-8b85-7a603794176b kind: Event lastTimestamp: "2026-04-16T14:13:31Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-7cc0c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-7cc0c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:13:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:13:31Z" name: isvc-logger-raw-7cc0c.18a6dbd5287ab74e namespace: kserve-ci-e2e-test resourceVersion: "17051" uid: 8d7702b5-52e5-43fd-8e4a-fe19ae13c4f5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:14:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-7cc0c namespace: kserve-ci-e2e-test resourceVersion: "17664" uid: 4abcbdd1-700e-42b5-8b85-7a603794176b kind: Event lastTimestamp: "2026-04-16T14:14:49Z" message: InferenceService [isvc-logger-raw-7cc0c] is Ready metadata: creationTimestamp: "2026-04-16T14:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:14:49Z" name: isvc-logger-raw-7cc0c.18a6dbe74a091255 namespace: kserve-ci-e2e-test resourceVersion: "17665" uid: d601ae97-7f9f-4741-a51b-6e7e30a57cf7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:17:27.172000Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z namespace: kserve-ci-e2e-test resourceVersion: "18930" uid: 04cd236f-53b4-4da3-90e7-c029b17aa260 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z to ip-10-0-141-131.ec2.internal metadata: creationTimestamp: "2026-04-16T14:17:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:17:27Z" name: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z.18a6dc0c01ca883a namespace: kserve-ci-e2e-test resourceVersion: "18935" uid: cb83dd54-1c51-40bb-a7fd-736c29865c88 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-66bdb995df-m8nk5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z namespace: kserve-ci-e2e-test resourceVersion: "18938" uid: 04cd236f-53b4-4da3-90e7-c029b17aa260 kind: Event lastTimestamp: "2026-04-16T14:17:27Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:17:27Z" name: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z.18a6dc0c1ad888bb namespace: kserve-ci-e2e-test resourceVersion: "18949" uid: afe880bd-cc4c-49f1-b159-cbbecf046444 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z namespace: kserve-ci-e2e-test resourceVersion: "18932" uid: 04cd236f-53b4-4da3-90e7-c029b17aa260 kind: Event lastTimestamp: "2026-04-16T14:17:27Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:16546eaa530f8a9a22c60a7ffa82a496f33cd9dfcf4989c8baef0968e8392589" already present on machine metadata: creationTimestamp: "2026-04-16T14:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:17:27Z" name: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z.18a6dc0c1c11f5f7 namespace: kserve-ci-e2e-test resourceVersion: "18951" uid: 54b39831-c1b0-4cd1-b58c-d2e6587c5aca reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z namespace: kserve-ci-e2e-test resourceVersion: "18932" uid: 04cd236f-53b4-4da3-90e7-c029b17aa260 kind: Event lastTimestamp: "2026-04-16T14:17:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:17:27Z" name: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z.18a6dc0c2056e82b namespace: kserve-ci-e2e-test resourceVersion: "18952" uid: 564d5906-7981-4a18-b4fb-d99cd4574727 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z namespace: kserve-ci-e2e-test resourceVersion: "18932" uid: 04cd236f-53b4-4da3-90e7-c029b17aa260 kind: Event lastTimestamp: "2026-04-16T14:17:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:17:27Z" name: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z.18a6dc0c20d28536 namespace: kserve-ci-e2e-test resourceVersion: "18953" uid: c8a64667-fa03-4ba4-90a3-d639cddb1eb8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z namespace: kserve-ci-e2e-test resourceVersion: "18932" uid: 04cd236f-53b4-4da3-90e7-c029b17aa260 kind: Event lastTimestamp: "2026-04-16T14:17:32Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1362" already present on machine metadata: creationTimestamp: "2026-04-16T14:17:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:17:32Z" name: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z.18a6dc0d2729e21b namespace: kserve-ci-e2e-test resourceVersion: "18991" uid: 3ee46424-c55a-4323-95ff-fbc8a49c5170 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z namespace: kserve-ci-e2e-test resourceVersion: "18932" uid: 04cd236f-53b4-4da3-90e7-c029b17aa260 kind: Event lastTimestamp: "2026-04-16T14:17:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:17:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:17:32Z" name: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z.18a6dc0d2a213932 namespace: kserve-ci-e2e-test resourceVersion: "18995" uid: 59305822-1e76-435c-84f8-ad2524c32369 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z namespace: kserve-ci-e2e-test resourceVersion: "18932" uid: 04cd236f-53b4-4da3-90e7-c029b17aa260 kind: Event lastTimestamp: "2026-04-16T14:17:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:17:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:17:32Z" name: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z.18a6dc0d2a854535 namespace: kserve-ci-e2e-test resourceVersion: "18996" uid: c0076599-cb5d-4648-b948-12567c742ecc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T14:17:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z namespace: kserve-ci-e2e-test resourceVersion: "18932" uid: 04cd236f-53b4-4da3-90e7-c029b17aa260 kind: Event lastTimestamp: "2026-04-16T14:19:04Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T14:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:04Z" name: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z.18a6dc0d63104cd6 namespace: kserve-ci-e2e-test resourceVersion: "19815" uid: 4317b712-736d-47ab-8765-9f2a30a51974 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z namespace: kserve-ci-e2e-test resourceVersion: "18932" uid: 04cd236f-53b4-4da3-90e7-c029b17aa260 kind: Event lastTimestamp: "2026-04-16T14:19:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:19:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:03Z" name: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z.18a6dc226cfa681b namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 7b2c6518-d802-4047-a2d9-a60a757c4b43 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-0dd324-predictor-5f56f6dd6f namespace: kserve-ci-e2e-test resourceVersion: "18920" uid: a78a9316-2760-4bc2-8540-52e05948e108 kind: Event lastTimestamp: "2026-04-16T14:17:27Z" message: 'Created pod: isvc-primary-0dd324-predictor-5f56f6dd6f-wvg7z' metadata: creationTimestamp: "2026-04-16T14:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:17:27Z" name: isvc-primary-0dd324-predictor-5f56f6dd6f.18a6dc0c0157ac1f namespace: kserve-ci-e2e-test resourceVersion: "18931" uid: 67cb0fca-486a-435d-ba96-14e3a1691715 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-0dd324-predictor namespace: kserve-ci-e2e-test resourceVersion: "18916" uid: d8801ee7-13fc-4700-83a6-14a792fd9200 kind: Event lastTimestamp: "2026-04-16T14:17:27Z" message: Scaled up replica set isvc-primary-0dd324-predictor-5f56f6dd6f from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:17:27Z" name: isvc-primary-0dd324-predictor.18a6dc0bfd126b0b namespace: kserve-ci-e2e-test resourceVersion: "18921" uid: 85b255a8-d11f-4120-9fd5-05fb4038c7d1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:17:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-0dd324-predictor namespace: kserve-ci-e2e-test resourceVersion: "18919" uid: 73d1f389-e59f-442b-a4ee-87aa2b81a0c8 kind: Event lastTimestamp: "2026-04-16T14:18:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:18:12Z" name: isvc-primary-0dd324-predictor.18a6dc0f7c22d081 namespace: kserve-ci-e2e-test resourceVersion: "19286" uid: 97e89475-cfe5-4d93-945a-4e29d1f5742d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:17:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-0dd324-predictor namespace: kserve-ci-e2e-test resourceVersion: "18919" uid: 73d1f389-e59f-442b-a4ee-87aa2b81a0c8 kind: Event lastTimestamp: "2026-04-16T14:18:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:18:12Z" name: isvc-primary-0dd324-predictor.18a6dc0f7cbb57a6 namespace: kserve-ci-e2e-test resourceVersion: "19287" uid: 469dfc10-918a-49b6-a745-3f41efbc1391 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:18:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-0dd324-predictor namespace: kserve-ci-e2e-test resourceVersion: "19074" uid: 73d1f389-e59f-442b-a4ee-87aa2b81a0c8 kind: Event lastTimestamp: "2026-04-16T14:18:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:18:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:18:57Z" name: isvc-primary-0dd324-predictor.18a6dc19f9d55324 namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: 99ea2af0-3dd7-4f0c-bd14-05f848cf1661 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:18:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-0dd324-predictor namespace: kserve-ci-e2e-test resourceVersion: "19074" uid: 73d1f389-e59f-442b-a4ee-87aa2b81a0c8 kind: Event lastTimestamp: "2026-04-16T14:18:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:18:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:18:57Z" name: isvc-primary-0dd324-predictor.18a6dc19fa3819fc namespace: kserve-ci-e2e-test resourceVersion: "19679" uid: 2db750fc-08e3-4e4e-bd82-fbad9ec83595 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-04-16T14:17:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-0dd324 namespace: kserve-ci-e2e-test resourceVersion: "18893" uid: e464743d-f097-45c7-a204-94d934d88aff kind: Event lastTimestamp: "2026-04-16T14:18:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:18:44Z" name: isvc-primary-0dd324.18a6dc0bfd1d1564 namespace: kserve-ci-e2e-test resourceVersion: "19519" uid: 13ccda7b-c1ef-4a10-9ad1-5571dcfb3761 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-0dd324 namespace: kserve-ci-e2e-test resourceVersion: "18893" uid: e464743d-f097-45c7-a204-94d934d88aff kind: Event lastTimestamp: "2026-04-16T14:17:27Z" message: 'Failed to update status for InferenceService "isvc-primary-0dd324": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-0dd324": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:17:27Z" name: isvc-primary-0dd324.18a6dc0bfdae6a51 namespace: kserve-ci-e2e-test resourceVersion: "18925" uid: a8ac720c-6626-4b50-9c9c-769eb7868fbf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-0dd324 namespace: kserve-ci-e2e-test resourceVersion: "18893" uid: e464743d-f097-45c7-a204-94d934d88aff kind: Event lastTimestamp: "2026-04-16T14:17:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-0dd324": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:17:27Z" name: isvc-primary-0dd324.18a6dc0bfdaeb8fc namespace: kserve-ci-e2e-test resourceVersion: "18927" uid: 75d38ef0-fba8-41fc-a22e-ffe24c5330f1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:18:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-0dd324 namespace: kserve-ci-e2e-test resourceVersion: "19517" uid: e464743d-f097-45c7-a204-94d934d88aff kind: Event lastTimestamp: "2026-04-16T14:18:44Z" message: InferenceService [isvc-primary-0dd324] is Ready metadata: creationTimestamp: "2026-04-16T14:18:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:18:44Z" name: isvc-primary-0dd324.18a6dc1def9c577d namespace: kserve-ci-e2e-test resourceVersion: "19518" uid: d3e86d5b-7011-40e6-8afc-66b0ea8e96cb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:08:15.754110Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd namespace: kserve-ci-e2e-test resourceVersion: "14165" uid: 2ec80429-b3eb-45df-abe8-084948810587 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd to ip-10-0-141-131.ec2.internal metadata: creationTimestamp: "2026-04-16T14:08:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:08:15Z" name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd.18a6db8b9eb93fa4 namespace: kserve-ci-e2e-test resourceVersion: "14172" uid: cf7bd33b-6f2a-4564-a1e7-40d07a26fb5c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-66bdb995df-m8nk5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd namespace: kserve-ci-e2e-test resourceVersion: "14175" uid: 2ec80429-b3eb-45df-abe8-084948810587 kind: Event lastTimestamp: "2026-04-16T14:08:16Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:08:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:08:16Z" name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd.18a6db8bb7f40473 namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: f00cf12c-27d1-4fa3-ba13-9d55f0663994 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 2ec80429-b3eb-45df-abe8-084948810587 kind: Event lastTimestamp: "2026-04-16T14:08:16Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:16546eaa530f8a9a22c60a7ffa82a496f33cd9dfcf4989c8baef0968e8392589" metadata: creationTimestamp: "2026-04-16T14:08:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:08:16Z" name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd.18a6db8bb92077cd namespace: kserve-ci-e2e-test resourceVersion: "14183" uid: c66a1063-c6a4-40ef-b1f6-1c9de011d786 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 2ec80429-b3eb-45df-abe8-084948810587 kind: Event lastTimestamp: "2026-04-16T14:08:20Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:16546eaa530f8a9a22c60a7ffa82a496f33cd9dfcf4989c8baef0968e8392589" in 4.607s (4.607s including waiting). Image size: 300955052 bytes.' metadata: creationTimestamp: "2026-04-16T14:08:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:08:20Z" name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd.18a6db8ccbc2e0d8 namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: 93d6829f-a845-4ef8-b908-f2ab04959487 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 2ec80429-b3eb-45df-abe8-084948810587 kind: Event lastTimestamp: "2026-04-16T14:08:20Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:08:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:08:20Z" name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd.18a6db8ccf2b2b80 namespace: kserve-ci-e2e-test resourceVersion: "14223" uid: b595bd4f-9a69-4b95-96fe-1475daf62aa5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 2ec80429-b3eb-45df-abe8-084948810587 kind: Event lastTimestamp: "2026-04-16T14:08:20Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:08:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:08:20Z" name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd.18a6db8ccf9ebce2 namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: 4bc10b80-bdaf-496f-83d8-3f33ccc1474c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 2ec80429-b3eb-45df-abe8-084948810587 kind: Event lastTimestamp: "2026-04-16T14:08:25Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1362" metadata: creationTimestamp: "2026-04-16T14:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:08:25Z" name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd.18a6db8ddc3df8f8 namespace: kserve-ci-e2e-test resourceVersion: "14259" uid: 764234ad-e735-45eb-95da-e775ef166eed reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 2ec80429-b3eb-45df-abe8-084948810587 kind: Event lastTimestamp: "2026-04-16T14:08:38Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1362" in 13.59s (13.59s including waiting). Image size: 1560613810 bytes.' metadata: creationTimestamp: "2026-04-16T14:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:08:38Z" name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd.18a6db91064b1391 namespace: kserve-ci-e2e-test resourceVersion: "14361" uid: e4e5753e-d2a7-413a-ba03-c9e2b3ab2227 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 2ec80429-b3eb-45df-abe8-084948810587 kind: Event lastTimestamp: "2026-04-16T14:08:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:08:39Z" name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd.18a6db91092740d9 namespace: kserve-ci-e2e-test resourceVersion: "14362" uid: 0b774408-cb22-4aa6-a975-8a4b271041aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 2ec80429-b3eb-45df-abe8-084948810587 kind: Event lastTimestamp: "2026-04-16T14:08:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:08:39Z" name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd.18a6db9109977f41 namespace: kserve-ci-e2e-test resourceVersion: "14363" uid: b85f287e-efca-4a75-8b37-1a9ed76cae40 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 2ec80429-b3eb-45df-abe8-084948810587 kind: Event lastTimestamp: "2026-04-16T14:08:39Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:cda273db4b40777305d78f4b7f8bde558e921e8f875775edae595cda2e7cfa4b" metadata: creationTimestamp: "2026-04-16T14:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:08:39Z" name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd.18a6db9109a44e01 namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 19dc02de-918d-4894-b4b5-ee7ea1370102 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 2ec80429-b3eb-45df-abe8-084948810587 kind: Event lastTimestamp: "2026-04-16T14:08:41Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:cda273db4b40777305d78f4b7f8bde558e921e8f875775edae595cda2e7cfa4b" in 2.585s (2.585s including waiting). Image size: 237894732 bytes.' metadata: creationTimestamp: "2026-04-16T14:08:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:08:41Z" name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd.18a6db91a3c5094e namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: b87fd341-6e1b-4b7c-8346-3edd4e1b3695 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 2ec80429-b3eb-45df-abe8-084948810587 kind: Event lastTimestamp: "2026-04-16T14:08:41Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-16T14:08:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:08:41Z" name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd.18a6db91a6a18287 namespace: kserve-ci-e2e-test resourceVersion: "14384" uid: 8feefc5d-1a5e-4281-99a5-a0b72177b5b0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 2ec80429-b3eb-45df-abe8-084948810587 kind: Event lastTimestamp: "2026-04-16T14:08:41Z" message: Started container agent metadata: creationTimestamp: "2026-04-16T14:08:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:08:41Z" name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd.18a6db91a70a055e namespace: kserve-ci-e2e-test resourceVersion: "14385" uid: 09421b05-00e6-4124-9da0-2dc2bc0a4270 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-16T14:08:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 2ec80429-b3eb-45df-abe8-084948810587 kind: Event lastTimestamp: "2026-04-16T14:10:14Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T14:08:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:14Z" name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd.18a6db91d587932b namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: 836472c3-b8d0-4431-9076-2584fd8422a8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T14:08:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 2ec80429-b3eb-45df-abe8-084948810587 kind: Event lastTimestamp: "2026-04-16T14:10:14Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T14:08:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:14Z" name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd.18a6db9211452c01 namespace: kserve-ci-e2e-test resourceVersion: "15191" uid: 6f5aa2b0-1746-4c62-a2d8-a72018217a83 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 2ec80429-b3eb-45df-abe8-084948810587 kind: Event lastTimestamp: "2026-04-16T14:09:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:09:50Z" name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd.18a6dba1c91d97ce namespace: kserve-ci-e2e-test resourceVersion: "14923" uid: 8eccf130-95a5-4412-b562-266e050fa9e0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 2ec80429-b3eb-45df-abe8-084948810587 kind: Event lastTimestamp: "2026-04-16T14:09:50Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-16T14:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:09:50Z" name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd.18a6dba1c91ecaea namespace: kserve-ci-e2e-test resourceVersion: "14926" uid: c18077f5-bd2d-467e-9134-61697ab70cc3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b namespace: kserve-ci-e2e-test resourceVersion: "14152" uid: 1924c543-7761-430f-ad38-a84cf81441f4 kind: Event lastTimestamp: "2026-04-16T14:08:15Z" message: 'Created pod: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b-2kdkd' metadata: creationTimestamp: "2026-04-16T14:08:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:08:15Z" name: isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b.18a6db8b9e42a0ac namespace: kserve-ci-e2e-test resourceVersion: "14167" uid: 754a3548-5914-47d6-8c7a-9fad559f38ed reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-6af14-predictor namespace: kserve-ci-e2e-test resourceVersion: "14151" uid: 481861f1-f412-40fa-8820-04c0d1f713c7 kind: Event lastTimestamp: "2026-04-16T14:08:15Z" message: Scaled up replica set isvc-raw-sklearn-batcher-6af14-predictor-848cbf7b6b from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:08:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:08:15Z" name: isvc-raw-sklearn-batcher-6af14-predictor.18a6db8b965f9a04 namespace: kserve-ci-e2e-test resourceVersion: "14153" uid: 78477395-a7e7-41e3-bb7f-34022e83085e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T14:08:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-6af14-predictor namespace: kserve-ci-e2e-test resourceVersion: "14158" uid: cf3c2f84-04d3-41db-b236-98e3e8df8407 kind: Event lastTimestamp: "2026-04-16T14:09:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:08:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:09:15Z" name: isvc-raw-sklearn-batcher-6af14-predictor.18a6db8f1ce3bbbb namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: fecaf345-cf6c-494a-b21b-d05541fc0ee0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T14:08:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-6af14-predictor namespace: kserve-ci-e2e-test resourceVersion: "14158" uid: cf3c2f84-04d3-41db-b236-98e3e8df8407 kind: Event lastTimestamp: "2026-04-16T14:09:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:08:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:09:15Z" name: isvc-raw-sklearn-batcher-6af14-predictor.18a6db8f1d50bcc4 namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: bc4c44ee-d2df-4b54-ade0-88bced9ddaf1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:09:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-6af14-predictor namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: cf3c2f84-04d3-41db-b236-98e3e8df8407 kind: Event lastTimestamp: "2026-04-16T14:09:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:09:45Z" name: isvc-raw-sklearn-batcher-6af14-predictor.18a6db9d1b6d18ef namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: b7f3761a-fa50-44ba-bd5a-9197f7eb51d8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:09:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-6af14-predictor namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: cf3c2f84-04d3-41db-b236-98e3e8df8407 kind: Event lastTimestamp: "2026-04-16T14:09:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:09:45Z" name: isvc-raw-sklearn-batcher-6af14-predictor.18a6db9d1bd67cc3 namespace: kserve-ci-e2e-test resourceVersion: "14858" uid: 053003fb-9609-4a97-8655-a2326e808fe4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T14:08:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-6af14 namespace: kserve-ci-e2e-test resourceVersion: "14141" uid: 54b28c5b-c272-4fd4-a77a-992052886049 kind: Event lastTimestamp: "2026-04-16T14:09:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:08:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:09:44Z" name: isvc-raw-sklearn-batcher-6af14.18a6db8b9ca3482a namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: 3fad1a9c-5ea9-4fcf-81f4-ecbfe84e3a17 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-6af14 namespace: kserve-ci-e2e-test resourceVersion: "14141" uid: 54b28c5b-c272-4fd4-a77a-992052886049 kind: Event lastTimestamp: "2026-04-16T14:08:15Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-6af14": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-6af14": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:08:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:08:15Z" name: isvc-raw-sklearn-batcher-6af14.18a6db8b9d0b691b namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: f6609cde-6f83-4446-a1c2-c636200c8794 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-6af14 namespace: kserve-ci-e2e-test resourceVersion: "14141" uid: 54b28c5b-c272-4fd4-a77a-992052886049 kind: Event lastTimestamp: "2026-04-16T14:08:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-6af14": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:08:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:08:15Z" name: isvc-raw-sklearn-batcher-6af14.18a6db8b9d0bb2f6 namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 975d832b-6ee3-420a-bed1-6f6266f3310e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-6af14 namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: 54b28c5b-c272-4fd4-a77a-992052886049 kind: Event lastTimestamp: "2026-04-16T14:09:44Z" message: InferenceService [isvc-raw-sklearn-batcher-6af14] is Ready metadata: creationTimestamp: "2026-04-16T14:09:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:09:44Z" name: isvc-raw-sklearn-batcher-6af14.18a6dba04a5f4fc3 namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: e168c1cc-016a-409e-ad0c-55072faa8418 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:18:47.352568Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-0dd324-predictor-66c78d4b56-8zdcw namespace: kserve-ci-e2e-test resourceVersion: "19573" uid: 4e49926c-52a9-466a-8bb0-4d8e3872cf1c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-0dd324-predictor-66c78d4b56-8zdcw to ip-10-0-141-131.ec2.internal metadata: creationTimestamp: "2026-04-16T14:18:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:18:47Z" name: isvc-secondary-0dd324-predictor-66c78d4b56-8zdcw.18a6dc1eacece334 namespace: kserve-ci-e2e-test resourceVersion: "19580" uid: 145ff311-7b7b-4822-9e83-ca80fe255dac reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-66bdb995df-m8nk5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:18:47Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-0dd324-predictor-66c78d4b56-8zdcw namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: 4e49926c-52a9-466a-8bb0-4d8e3872cf1c kind: Event lastTimestamp: "2026-04-16T14:18:47Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:18:47Z" name: isvc-secondary-0dd324-predictor-66c78d4b56-8zdcw.18a6dc1ec63a2275 namespace: kserve-ci-e2e-test resourceVersion: "19589" uid: 7401bc30-2af8-48d1-b973-ed06e8c4e370 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:18:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-0dd324-predictor-66c78d4b56-8zdcw namespace: kserve-ci-e2e-test resourceVersion: "19576" uid: 4e49926c-52a9-466a-8bb0-4d8e3872cf1c kind: Event lastTimestamp: "2026-04-16T14:18:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:16546eaa530f8a9a22c60a7ffa82a496f33cd9dfcf4989c8baef0968e8392589" already present on machine metadata: creationTimestamp: "2026-04-16T14:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:18:52Z" name: isvc-secondary-0dd324-predictor-66c78d4b56-8zdcw.18a6dc1ec78f12b1 namespace: kserve-ci-e2e-test resourceVersion: "19629" uid: b1233daa-29b1-47b6-942a-cc49f5027f03 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:18:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-0dd324-predictor-66c78d4b56-8zdcw namespace: kserve-ci-e2e-test resourceVersion: "19576" uid: 4e49926c-52a9-466a-8bb0-4d8e3872cf1c kind: Event lastTimestamp: "2026-04-16T14:18:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:18:52Z" name: isvc-secondary-0dd324-predictor-66c78d4b56-8zdcw.18a6dc1eca88f305 namespace: kserve-ci-e2e-test resourceVersion: "19633" uid: 5ad8d786-63d9-43ee-868b-2751278db8ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:18:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-0dd324-predictor-66c78d4b56-8zdcw namespace: kserve-ci-e2e-test resourceVersion: "19576" uid: 4e49926c-52a9-466a-8bb0-4d8e3872cf1c kind: Event lastTimestamp: "2026-04-16T14:18:52Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:18:52Z" name: isvc-secondary-0dd324-predictor-66c78d4b56-8zdcw.18a6dc1ecb041bdb namespace: kserve-ci-e2e-test resourceVersion: "19634" uid: e37c3847-1974-4199-bbae-0aefafa4ea52 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:18:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-0dd324-predictor-66c78d4b56-8zdcw namespace: kserve-ci-e2e-test resourceVersion: "19576" uid: 4e49926c-52a9-466a-8bb0-4d8e3872cf1c kind: Event lastTimestamp: "2026-04-16T14:18:55Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-0dd324-predictor-66c78d4b56-8zdcw_kserve-ci-e2e-test(4e49926c-52a9-466a-8bb0-4d8e3872cf1c) metadata: creationTimestamp: "2026-04-16T14:18:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:18:55Z" name: isvc-secondary-0dd324-predictor-66c78d4b56-8zdcw.18a6dc208acf8eb9 namespace: kserve-ci-e2e-test resourceVersion: "19656" uid: 4a381236-f18a-4c02-8cde-a2d0e4a781da reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:18:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-0dd324-predictor-66c78d4b56 namespace: kserve-ci-e2e-test resourceVersion: "19564" uid: 698fc8d5-442e-4bbf-bb1f-955262fa18c0 kind: Event lastTimestamp: "2026-04-16T14:18:47Z" message: 'Created pod: isvc-secondary-0dd324-predictor-66c78d4b56-8zdcw' metadata: creationTimestamp: "2026-04-16T14:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:18:47Z" name: isvc-secondary-0dd324-predictor-66c78d4b56.18a6dc1eac631753 namespace: kserve-ci-e2e-test resourceVersion: "19575" uid: 40b2a07f-991d-41d9-9795-166ecb9e2d9f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:18:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-0dd324-predictor namespace: kserve-ci-e2e-test resourceVersion: "19562" uid: deaf13a0-757b-4e2f-a77e-fb0f663594ab kind: Event lastTimestamp: "2026-04-16T14:18:47Z" message: Scaled up replica set isvc-secondary-0dd324-predictor-66c78d4b56 from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:18:47Z" name: isvc-secondary-0dd324-predictor.18a6dc1ea97cf044 namespace: kserve-ci-e2e-test resourceVersion: "19566" uid: 844e8540-ee7b-40ea-b8ce-96670b108bd6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-0dd324-predictor namespace: kserve-ci-e2e-test resourceVersion: "19563" uid: 67120a12-b30b-4fc1-aa1b-16af74d6fc80 kind: Event lastTimestamp: "2026-04-16T14:19:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:19:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:19:02Z" name: isvc-secondary-0dd324-predictor.18a6dc22280104e7 namespace: kserve-ci-e2e-test resourceVersion: "19716" uid: ede498da-e1bb-4ab0-95d7-37b14e70a918 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-0dd324-predictor namespace: kserve-ci-e2e-test resourceVersion: "19563" uid: 67120a12-b30b-4fc1-aa1b-16af74d6fc80 kind: Event lastTimestamp: "2026-04-16T14:19:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:19:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:19:02Z" name: isvc-secondary-0dd324-predictor.18a6dc22288385af namespace: kserve-ci-e2e-test resourceVersion: "19718" uid: d79642cd-3380-4064-97fb-f5101f0f11a3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T14:18:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-0dd324 namespace: kserve-ci-e2e-test resourceVersion: "19554" uid: 97bb8341-0937-4dd1-a475-c999d75cdf51 kind: Event lastTimestamp: "2026-04-16T14:18:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:18:56Z" name: isvc-secondary-0dd324.18a6dc1ea959041c namespace: kserve-ci-e2e-test resourceVersion: "19668" uid: 7c11240f-7187-4121-8539-3f4d759ec7ea reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:18:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-0dd324 namespace: kserve-ci-e2e-test resourceVersion: "19554" uid: 97bb8341-0937-4dd1-a475-c999d75cdf51 kind: Event lastTimestamp: "2026-04-16T14:18:47Z" message: 'Failed to update status for InferenceService "isvc-secondary-0dd324": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-0dd324": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:18:47Z" name: isvc-secondary-0dd324.18a6dc1ea9c60ef0 namespace: kserve-ci-e2e-test resourceVersion: "19568" uid: a4268114-42eb-422d-aa13-ab1bb30bc437 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:18:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-0dd324 namespace: kserve-ci-e2e-test resourceVersion: "19554" uid: 97bb8341-0937-4dd1-a475-c999d75cdf51 kind: Event lastTimestamp: "2026-04-16T14:18:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-0dd324": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:18:47Z" name: isvc-secondary-0dd324.18a6dc1ea9c65f12 namespace: kserve-ci-e2e-test resourceVersion: "19569" uid: 793cb490-4d16-4548-8762-4b6f3755e8b7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:09:51.053957Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q namespace: kserve-ci-e2e-test resourceVersion: "14943" uid: 4bc59a57-592a-427d-b4fb-53fad0ca5032 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q to ip-10-0-141-131.ec2.internal metadata: creationTimestamp: "2026-04-16T14:09:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:09:51Z" name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q.18a6dba1cf0980a5 namespace: kserve-ci-e2e-test resourceVersion: "14955" uid: b6d86595-3606-4d91-b091-f734d4143ead reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-66bdb995df-m8nk5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q namespace: kserve-ci-e2e-test resourceVersion: "14959" uid: 4bc59a57-592a-427d-b4fb-53fad0ca5032 kind: Event lastTimestamp: "2026-04-16T14:09:51Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:09:51Z" name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q.18a6dba1e856a597 namespace: kserve-ci-e2e-test resourceVersion: "14989" uid: fca1982c-af51-48c4-ac86-f9fab131d898 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: 4bc59a57-592a-427d-b4fb-53fad0ca5032 kind: Event lastTimestamp: "2026-04-16T14:09:51Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:16546eaa530f8a9a22c60a7ffa82a496f33cd9dfcf4989c8baef0968e8392589" already present on machine metadata: creationTimestamp: "2026-04-16T14:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:09:51Z" name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q.18a6dba1e9e017df namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: a4b931d4-adc0-430b-bb5c-0e237f54472f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: 4bc59a57-592a-427d-b4fb-53fad0ca5032 kind: Event lastTimestamp: "2026-04-16T14:09:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:09:51Z" name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q.18a6dba1ece59e51 namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 1c9bfd9a-4f29-4825-b2ac-ee4a042ddcc2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: 4bc59a57-592a-427d-b4fb-53fad0ca5032 kind: Event lastTimestamp: "2026-04-16T14:09:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:09:51Z" name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q.18a6dba1ed70769c namespace: kserve-ci-e2e-test resourceVersion: "14994" uid: 6d47375c-9c21-410d-a527-f902860952b3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: 4bc59a57-592a-427d-b4fb-53fad0ca5032 kind: Event lastTimestamp: "2026-04-16T14:09:55Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1362" already present on machine metadata: creationTimestamp: "2026-04-16T14:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:09:55Z" name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q.18a6dba2e20eb086 namespace: kserve-ci-e2e-test resourceVersion: "15046" uid: d3bf172f-9a1c-4202-b26f-04aade9bbd04 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: 4bc59a57-592a-427d-b4fb-53fad0ca5032 kind: Event lastTimestamp: "2026-04-16T14:09:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:09:55Z" name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q.18a6dba2e557540b namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: 6496cbdd-d516-428e-8998-d1731b880da7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: 4bc59a57-592a-427d-b4fb-53fad0ca5032 kind: Event lastTimestamp: "2026-04-16T14:09:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:09:55Z" name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q.18a6dba2e5b4c28b namespace: kserve-ci-e2e-test resourceVersion: "15052" uid: 4281765f-aea9-4bf1-8012-c3afd9be0e41 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T14:09:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: 4bc59a57-592a-427d-b4fb-53fad0ca5032 kind: Event lastTimestamp: "2026-04-16T14:11:44Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T14:09:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:44Z" name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q.18a6dba31e20ba60 namespace: kserve-ci-e2e-test resourceVersion: "16061" uid: 35444a56-6b15-44d1-a066-974cad98ade6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: 4bc59a57-592a-427d-b4fb-53fad0ca5032 kind: Event lastTimestamp: "2026-04-16T14:11:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:41Z" name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q.18a6dbbb7b32e5c5 namespace: kserve-ci-e2e-test resourceVersion: "15950" uid: 60ee5b78-eaec-4554-8f30-9f550e650bed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664 namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: a53a7727-dd27-46b0-98fa-882cc51047f4 kind: Event lastTimestamp: "2026-04-16T14:09:51Z" message: 'Created pod: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664-sg94q' metadata: creationTimestamp: "2026-04-16T14:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:09:51Z" name: isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664.18a6dba1cddb2d1c namespace: kserve-ci-e2e-test resourceVersion: "14947" uid: 39d8228e-5d69-4d38-82a1-02c29c820fbc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-bd7db-predictor namespace: kserve-ci-e2e-test resourceVersion: "14912" uid: 6e49276e-9aa5-4878-b375-cc76b6aff290 kind: Event lastTimestamp: "2026-04-16T14:09:50Z" message: Scaled up replica set isvc-sklearn-graph-raw-bd7db-predictor-fd55b8664 from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:09:50Z" name: isvc-sklearn-graph-raw-bd7db-predictor.18a6dba1c8a8b436 namespace: kserve-ci-e2e-test resourceVersion: "14920" uid: 74dcc0cb-99dd-4baa-bd8e-d10adc52c2cb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:10:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-bd7db-predictor namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: 361d3f7b-761f-4f30-9922-2687d71fb35c kind: Event lastTimestamp: "2026-04-16T14:10:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:10:20Z" name: isvc-sklearn-graph-raw-bd7db-predictor.18a6dba547732762 namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: adde7e40-4385-4aee-947c-0dd264b9bc52 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:10:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-bd7db-predictor namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: 361d3f7b-761f-4f30-9922-2687d71fb35c kind: Event lastTimestamp: "2026-04-16T14:10:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:10:20Z" name: isvc-sklearn-graph-raw-bd7db-predictor.18a6dba547d6797e namespace: kserve-ci-e2e-test resourceVersion: "15250" uid: 5bf883c5-0467-4709-9ba1-443a90968056 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T14:10:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-bd7db-predictor namespace: kserve-ci-e2e-test resourceVersion: "15132" uid: 361d3f7b-761f-4f30-9922-2687d71fb35c kind: Event lastTimestamp: "2026-04-16T14:11:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:11:36Z" name: isvc-sklearn-graph-raw-bd7db-predictor.18a6dbac45d1cefc namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: 1b29db3f-532e-4001-8e81-e0aa778b7f26 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T14:10:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-bd7db-predictor namespace: kserve-ci-e2e-test resourceVersion: "15132" uid: 361d3f7b-761f-4f30-9922-2687d71fb35c kind: Event lastTimestamp: "2026-04-16T14:11:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:11:36Z" name: isvc-sklearn-graph-raw-bd7db-predictor.18a6dbac461d42df namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: 565a094f-8742-4305-915f-8c696f2a9784 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T14:09:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-bd7db namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 373825a9-f510-4a51-be0a-cb7c72c0152c kind: Event lastTimestamp: "2026-04-16T14:11:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:11:14Z" name: isvc-sklearn-graph-raw-bd7db.18a6dba1c8afbafb namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: dac5121a-508c-4354-ad7c-71c5b829d90b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-bd7db namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 373825a9-f510-4a51-be0a-cb7c72c0152c kind: Event lastTimestamp: "2026-04-16T14:09:50Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-bd7db": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-bd7db": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:09:50Z" name: isvc-sklearn-graph-raw-bd7db.18a6dba1c93f76b8 namespace: kserve-ci-e2e-test resourceVersion: "14925" uid: 5d9978d9-6518-4c4f-8763-9a211d44e606 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-bd7db namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: 373825a9-f510-4a51-be0a-cb7c72c0152c kind: Event lastTimestamp: "2026-04-16T14:11:14Z" message: InferenceService [isvc-sklearn-graph-raw-bd7db] is Ready metadata: creationTimestamp: "2026-04-16T14:11:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:11:14Z" name: isvc-sklearn-graph-raw-bd7db.18a6dbb53b16d22d namespace: kserve-ci-e2e-test resourceVersion: "15658" uid: 6215408d-2b32-4079-af00-5b9fd7823293 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:11:41.354594Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5 namespace: kserve-ci-e2e-test resourceVersion: "15957" uid: 78c543cb-924e-400d-a144-85d5eedf964f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5 to ip-10-0-141-131.ec2.internal metadata: creationTimestamp: "2026-04-16T14:11:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:11:41Z" name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5.18a6dbbb7d7805a2 namespace: kserve-ci-e2e-test resourceVersion: "15967" uid: d9f90d23-1a35-4e0a-8594-96c7fdf29feb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-66bdb995df-m8nk5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:41Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5 namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: 78c543cb-924e-400d-a144-85d5eedf964f kind: Event lastTimestamp: "2026-04-16T14:11:41Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:11:41Z" name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5.18a6dbbb96de738d namespace: kserve-ci-e2e-test resourceVersion: "16018" uid: 2949d92c-5b6d-4ad1-a378-0de14a613a33 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5 namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 78c543cb-924e-400d-a144-85d5eedf964f kind: Event lastTimestamp: "2026-04-16T14:11:41Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:16546eaa530f8a9a22c60a7ffa82a496f33cd9dfcf4989c8baef0968e8392589" already present on machine metadata: creationTimestamp: "2026-04-16T14:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:41Z" name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5.18a6dbbb983a7ffb namespace: kserve-ci-e2e-test resourceVersion: "16020" uid: dd81aa39-929a-4825-a662-03be74fa42eb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5 namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 78c543cb-924e-400d-a144-85d5eedf964f kind: Event lastTimestamp: "2026-04-16T14:11:41Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:41Z" name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5.18a6dbbb9ccd172f namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: d30e10a7-c4e9-423f-88d4-8c2442831f8b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5 namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 78c543cb-924e-400d-a144-85d5eedf964f kind: Event lastTimestamp: "2026-04-16T14:11:41Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:41Z" name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5.18a6dbbb9d7e7634 namespace: kserve-ci-e2e-test resourceVersion: "16023" uid: 74dd92c9-abdb-406c-a061-b2e3285596d5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5 namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 78c543cb-924e-400d-a144-85d5eedf964f kind: Event lastTimestamp: "2026-04-16T14:11:46Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1362" already present on machine metadata: creationTimestamp: "2026-04-16T14:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:46Z" name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5.18a6dbbc9446b449 namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: 860b7ce1-0e70-43c9-bce5-8e522a103f7c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5 namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 78c543cb-924e-400d-a144-85d5eedf964f kind: Event lastTimestamp: "2026-04-16T14:11:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:46Z" name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5.18a6dbbc9881b5ad namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: f20b6c56-1a9e-426b-a34f-f58e7ac04702 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5 namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 78c543cb-924e-400d-a144-85d5eedf964f kind: Event lastTimestamp: "2026-04-16T14:11:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:46Z" name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5.18a6dbbc992d3f05 namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: eda376fa-abe1-42a2-ace9-2f4121670239 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T14:11:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5 namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 78c543cb-924e-400d-a144-85d5eedf964f kind: Event lastTimestamp: "2026-04-16T14:12:48Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T14:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:12:48Z" name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5.18a6dbbcd049ae92 namespace: kserve-ci-e2e-test resourceVersion: "16558" uid: 78e64c8b-6124-4490-a61b-693c19596b0f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5 namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 78c543cb-924e-400d-a144-85d5eedf964f kind: Event lastTimestamp: "2026-04-16T14:13:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:21Z" name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5.18a6dbd2d5105a4b namespace: kserve-ci-e2e-test resourceVersion: "16910" uid: 7c60db08-fb31-48e2-a892-2573ea1242b3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: 50dfbe81-e235-4a28-9db4-91c489da5075 kind: Event lastTimestamp: "2026-04-16T14:11:41Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d-vpcm5' metadata: creationTimestamp: "2026-04-16T14:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:11:41Z" name: isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d.18a6dbbb7cc68a13 namespace: kserve-ci-e2e-test resourceVersion: "15959" uid: 1e798d76-bcc7-4c2a-b2c1-b9c6a6c2c4e4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-1d91c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15924" uid: acd3975c-4c77-4e41-b7ab-494df2e047e1 kind: Event lastTimestamp: "2026-04-16T14:11:41Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-1d91c-predictor-76b8b86b6d from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:11:41Z" name: isvc-sklearn-graph-raw-hpa-1d91c-predictor.18a6dbbb78141d4a namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: f9942d47-1f89-4986-b746-1aaaa679145d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:11:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-1d91c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15926" uid: fcc9f974-c925-4861-af9b-6cf217b1c825 kind: Event lastTimestamp: "2026-04-16T14:12:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:12:26Z" name: isvc-sklearn-graph-raw-hpa-1d91c-predictor.18a6dbbef6925879 namespace: kserve-ci-e2e-test resourceVersion: "16392" uid: 2cd8df44-e25d-4189-8d9c-b20a7ab06cdc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:11:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-1d91c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15926" uid: fcc9f974-c925-4861-af9b-6cf217b1c825 kind: Event lastTimestamp: "2026-04-16T14:12:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:12:26Z" name: isvc-sklearn-graph-raw-hpa-1d91c-predictor.18a6dbbef6f9682e namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 3cd52538-c436-46b9-9d26-39c9cdcf54b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:12:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-1d91c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: fcc9f974-c925-4861-af9b-6cf217b1c825 kind: Event lastTimestamp: "2026-04-16T14:13:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:12:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:11Z" name: isvc-sklearn-graph-raw-hpa-1d91c-predictor.18a6dbc97443c8fa namespace: kserve-ci-e2e-test resourceVersion: "16753" uid: 5a203b19-f2db-4b6b-9e19-b3ab5d0c8f28 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:12:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-1d91c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: fcc9f974-c925-4861-af9b-6cf217b1c825 kind: Event lastTimestamp: "2026-04-16T14:13:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:12:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:11Z" name: isvc-sklearn-graph-raw-hpa-1d91c-predictor.18a6dbc974a45e33 namespace: kserve-ci-e2e-test resourceVersion: "16758" uid: 374bc332-764d-4dfa-a712-49332c5d081c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T14:11:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-1d91c namespace: kserve-ci-e2e-test resourceVersion: "15918" uid: 3c953d92-12cb-49ab-bf61-eec8bd17bf2c kind: Event lastTimestamp: "2026-04-16T14:12:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:12:58Z" name: isvc-sklearn-graph-raw-hpa-1d91c.18a6dbbb77ecfa67 namespace: kserve-ci-e2e-test resourceVersion: "16653" uid: dee9ee0f-7487-4dd2-8d09-3ea19f813c90 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-1d91c namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: 3c953d92-12cb-49ab-bf61-eec8bd17bf2c kind: Event lastTimestamp: "2026-04-16T14:12:58Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-1d91c] is Ready metadata: creationTimestamp: "2026-04-16T14:12:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:12:58Z" name: isvc-sklearn-graph-raw-hpa-1d91c.18a6dbcd5c832fbe namespace: kserve-ci-e2e-test resourceVersion: "16652" uid: 101d833d-91ff-485e-ad06-5449df08cdf1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:14:56.910190Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv namespace: kserve-ci-e2e-test resourceVersion: "17758" uid: fc9c1d1d-5ded-4e1a-b096-56ce46b255e1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv to ip-10-0-141-131.ec2.internal metadata: creationTimestamp: "2026-04-16T14:14:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:14:56Z" name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv.18a6dbe9057d35c2 namespace: kserve-ci-e2e-test resourceVersion: "17762" uid: 2ea83074-f95c-47f5-abd5-a3369d26bf62 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-66bdb995df-m8nk5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:14:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv namespace: kserve-ci-e2e-test resourceVersion: "17766" uid: fc9c1d1d-5ded-4e1a-b096-56ce46b255e1 kind: Event lastTimestamp: "2026-04-16T14:14:57Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:14:57Z" name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv.18a6dbe91e67b469 namespace: kserve-ci-e2e-test resourceVersion: "17790" uid: 881c3c3a-a76d-43de-a86e-bda0f3670e76 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv namespace: kserve-ci-e2e-test resourceVersion: "17760" uid: fc9c1d1d-5ded-4e1a-b096-56ce46b255e1 kind: Event lastTimestamp: "2026-04-16T14:14:57Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:16546eaa530f8a9a22c60a7ffa82a496f33cd9dfcf4989c8baef0968e8392589" already present on machine metadata: creationTimestamp: "2026-04-16T14:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:14:57Z" name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv.18a6dbe91f9a4c2d namespace: kserve-ci-e2e-test resourceVersion: "17792" uid: d70571d7-1cb5-4052-be6b-47c0510615f5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv namespace: kserve-ci-e2e-test resourceVersion: "17760" uid: fc9c1d1d-5ded-4e1a-b096-56ce46b255e1 kind: Event lastTimestamp: "2026-04-16T14:14:57Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:14:57Z" name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv.18a6dbe9231d9a58 namespace: kserve-ci-e2e-test resourceVersion: "17793" uid: 14c848fb-db7e-4764-a74b-95e8b19e0baf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv namespace: kserve-ci-e2e-test resourceVersion: "17760" uid: fc9c1d1d-5ded-4e1a-b096-56ce46b255e1 kind: Event lastTimestamp: "2026-04-16T14:14:57Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:14:57Z" name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv.18a6dbe92391cd51 namespace: kserve-ci-e2e-test resourceVersion: "17794" uid: 1bea8d4d-81cc-4e7f-9ebe-70bcb869e169 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv namespace: kserve-ci-e2e-test resourceVersion: "17760" uid: fc9c1d1d-5ded-4e1a-b096-56ce46b255e1 kind: Event lastTimestamp: "2026-04-16T14:15:01Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1362" already present on machine metadata: creationTimestamp: "2026-04-16T14:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:15:01Z" name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv.18a6dbea1fa9abb7 namespace: kserve-ci-e2e-test resourceVersion: "17839" uid: 055acd95-89c8-4648-a155-77f322848a9e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv namespace: kserve-ci-e2e-test resourceVersion: "17760" uid: fc9c1d1d-5ded-4e1a-b096-56ce46b255e1 kind: Event lastTimestamp: "2026-04-16T14:15:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:15:01Z" name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv.18a6dbea22c504e6 namespace: kserve-ci-e2e-test resourceVersion: "17843" uid: 011e6516-fb78-4e79-8aaf-8ec3e86d9543 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv namespace: kserve-ci-e2e-test resourceVersion: "17760" uid: fc9c1d1d-5ded-4e1a-b096-56ce46b255e1 kind: Event lastTimestamp: "2026-04-16T14:15:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:15:01Z" name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv.18a6dbea233c714e namespace: kserve-ci-e2e-test resourceVersion: "17844" uid: 58183a51-44a0-43d1-b544-6feb5c72062a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 16 eventTime: null firstTimestamp: "2026-04-16T14:15:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv namespace: kserve-ci-e2e-test resourceVersion: "17760" uid: fc9c1d1d-5ded-4e1a-b096-56ce46b255e1 kind: Event lastTimestamp: "2026-04-16T14:17:31Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T14:15:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:17:31Z" name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv.18a6dbea5b95b835 namespace: kserve-ci-e2e-test resourceVersion: "18986" uid: 3d2023a4-221b-4b1b-8d62-9db1472adb76 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv namespace: kserve-ci-e2e-test resourceVersion: "17760" uid: fc9c1d1d-5ded-4e1a-b096-56ce46b255e1 kind: Event lastTimestamp: "2026-04-16T14:17:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:17:27Z" name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv.18a6dc0bfc839854 namespace: kserve-ci-e2e-test resourceVersion: "18918" uid: 9cdced86-fe8a-45dd-9983-380695567414 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:14:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66 namespace: kserve-ci-e2e-test resourceVersion: "17732" uid: 64fd1d83-7818-4b6f-a404-8a1ac9f87dc6 kind: Event lastTimestamp: "2026-04-16T14:14:56Z" message: 'Created pod: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66-whqrv' metadata: creationTimestamp: "2026-04-16T14:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:14:56Z" name: isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66.18a6dbe904ebe9b2 namespace: kserve-ci-e2e-test resourceVersion: "17759" uid: b053c2e2-1f7f-4320-bcaa-ede94e8b163b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:14:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-9b575-predictor namespace: kserve-ci-e2e-test resourceVersion: "17727" uid: 9b9c470d-3f20-4954-bbdb-eccc89e30f2a kind: Event lastTimestamp: "2026-04-16T14:14:56Z" message: Scaled up replica set isvc-sklearn-scale-raw-9b575-predictor-6fd5844c66 from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:14:56Z" name: isvc-sklearn-scale-raw-9b575-predictor.18a6dbe901c0447f namespace: kserve-ci-e2e-test resourceVersion: "17737" uid: 87c413b3-789c-41f6-9a35-1ccc923b32f7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:15:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-9b575-predictor namespace: kserve-ci-e2e-test resourceVersion: "17731" uid: aa13e9ab-ca19-4db6-a504-14977417d77f kind: Event lastTimestamp: "2026-04-16T14:15:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:15:26Z" name: isvc-sklearn-scale-raw-9b575-predictor.18a6dbec80b444a3 namespace: kserve-ci-e2e-test resourceVersion: "18025" uid: 1ee15dcc-cb7c-4a55-9cb6-9b719b3b4453 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:15:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-9b575-predictor namespace: kserve-ci-e2e-test resourceVersion: "17731" uid: aa13e9ab-ca19-4db6-a504-14977417d77f kind: Event lastTimestamp: "2026-04-16T14:15:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:15:26Z" name: isvc-sklearn-scale-raw-9b575-predictor.18a6dbec81186304 namespace: kserve-ci-e2e-test resourceVersion: "18026" uid: 85788d2c-826b-4d6b-969c-6c27c4352bd4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T14:15:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-9b575-predictor namespace: kserve-ci-e2e-test resourceVersion: "17923" uid: aa13e9ab-ca19-4db6-a504-14977417d77f kind: Event lastTimestamp: "2026-04-16T14:17:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:17:27Z" name: isvc-sklearn-scale-raw-9b575-predictor.18a6dbf37f10b3fa namespace: kserve-ci-e2e-test resourceVersion: "18891" uid: 49bb1c05-ea69-4427-a5e2-1c126a5a77f1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T14:15:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-9b575-predictor namespace: kserve-ci-e2e-test resourceVersion: "17923" uid: aa13e9ab-ca19-4db6-a504-14977417d77f kind: Event lastTimestamp: "2026-04-16T14:17:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:17:27Z" name: isvc-sklearn-scale-raw-9b575-predictor.18a6dbf37f7a0bc6 namespace: kserve-ci-e2e-test resourceVersion: "18892" uid: 0be441d3-c208-4336-a38a-501d5c3f3ed9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T14:14:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-9b575 namespace: kserve-ci-e2e-test resourceVersion: "17719" uid: 128c9a0d-9d38-42c7-8063-ce5aa84b6768 kind: Event lastTimestamp: "2026-04-16T14:17:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:17:21Z" name: isvc-sklearn-scale-raw-9b575.18a6dbe901be3bed namespace: kserve-ci-e2e-test resourceVersion: "18846" uid: 6221e9b3-661c-4ede-a51d-5fd7b56f7614 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:14:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-9b575 namespace: kserve-ci-e2e-test resourceVersion: "17719" uid: 128c9a0d-9d38-42c7-8063-ce5aa84b6768 kind: Event lastTimestamp: "2026-04-16T14:14:56Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-9b575": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-9b575": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:14:56Z" name: isvc-sklearn-scale-raw-9b575.18a6dbe90279c169 namespace: kserve-ci-e2e-test resourceVersion: "17743" uid: ffb380ea-bb56-4577-97de-65d22dd3968b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-9b575 namespace: kserve-ci-e2e-test resourceVersion: "18844" uid: 128c9a0d-9d38-42c7-8063-ce5aa84b6768 kind: Event lastTimestamp: "2026-04-16T14:17:21Z" message: InferenceService [isvc-sklearn-scale-raw-9b575] is Ready metadata: creationTimestamp: "2026-04-16T14:17:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:17:21Z" name: isvc-sklearn-scale-raw-9b575.18a6dc0aad5b4de6 namespace: kserve-ci-e2e-test resourceVersion: "18845" uid: 424e6755-e12d-4111-b16f-9d1f2d5bec64 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:09:51.156422Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v namespace: kserve-ci-e2e-test resourceVersion: "14964" uid: 88fe0cd5-1e65-4c38-b7b0-a35284232513 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v to ip-10-0-141-131.ec2.internal metadata: creationTimestamp: "2026-04-16T14:09:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:09:51Z" name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v.18a6dba1d5250482 namespace: kserve-ci-e2e-test resourceVersion: "14973" uid: 11dfc07a-59ea-44d8-a276-b69e6663cc84 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-66bdb995df-m8nk5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v namespace: kserve-ci-e2e-test resourceVersion: "14979" uid: 88fe0cd5-1e65-4c38-b7b0-a35284232513 kind: Event lastTimestamp: "2026-04-16T14:09:51Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:09:51Z" name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v.18a6dba1eec3557e namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: e3d41e16-2c48-41b0-8de9-5c85087c3244 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v namespace: kserve-ci-e2e-test resourceVersion: "14967" uid: 88fe0cd5-1e65-4c38-b7b0-a35284232513 kind: Event lastTimestamp: "2026-04-16T14:09:51Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:16546eaa530f8a9a22c60a7ffa82a496f33cd9dfcf4989c8baef0968e8392589" already present on machine metadata: creationTimestamp: "2026-04-16T14:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:09:51Z" name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v.18a6dba1f0b8a3d9 namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: 70fe75da-d2a6-4a7e-b907-45fdb55f8b30 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v namespace: kserve-ci-e2e-test resourceVersion: "14967" uid: 88fe0cd5-1e65-4c38-b7b0-a35284232513 kind: Event lastTimestamp: "2026-04-16T14:09:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:09:51Z" name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v.18a6dba1f3b2aad3 namespace: kserve-ci-e2e-test resourceVersion: "14999" uid: e61a6b24-e08a-4bc9-a75c-bb64b58ef0d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v namespace: kserve-ci-e2e-test resourceVersion: "14967" uid: 88fe0cd5-1e65-4c38-b7b0-a35284232513 kind: Event lastTimestamp: "2026-04-16T14:09:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:09:51Z" name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v.18a6dba1f422793a namespace: kserve-ci-e2e-test resourceVersion: "15002" uid: ebe714c4-55a2-4eb5-bfa3-1282486c2cf8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v namespace: kserve-ci-e2e-test resourceVersion: "14967" uid: 88fe0cd5-1e65-4c38-b7b0-a35284232513 kind: Event lastTimestamp: "2026-04-16T14:09:55Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-16T14:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:09:55Z" name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v.18a6dba2e1e57ea0 namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 2e893cf5-e742-46dd-9dbe-0fb7d0eb4b29 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v namespace: kserve-ci-e2e-test resourceVersion: "14967" uid: 88fe0cd5-1e65-4c38-b7b0-a35284232513 kind: Event lastTimestamp: "2026-04-16T14:10:16Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 20.671s (20.671s including waiting). Image size: 1306229499 bytes.' metadata: creationTimestamp: "2026-04-16T14:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:16Z" name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v.18a6dba7b204fb63 namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: 6b0b9fb9-d497-47a5-b724-52a247bf7c31 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v namespace: kserve-ci-e2e-test resourceVersion: "14967" uid: 88fe0cd5-1e65-4c38-b7b0-a35284232513 kind: Event lastTimestamp: "2026-04-16T14:10:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:16Z" name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v.18a6dba7b564644b namespace: kserve-ci-e2e-test resourceVersion: "15211" uid: 79613fd2-dcb0-46b4-b193-a89aea96140c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v namespace: kserve-ci-e2e-test resourceVersion: "14967" uid: 88fe0cd5-1e65-4c38-b7b0-a35284232513 kind: Event lastTimestamp: "2026-04-16T14:10:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:16Z" name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v.18a6dba7b5f7eaa3 namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: ca471e7c-7f15-4ad4-9148-7d66f334dfa4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T14:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v namespace: kserve-ci-e2e-test resourceVersion: "14967" uid: 88fe0cd5-1e65-4c38-b7b0-a35284232513 kind: Event lastTimestamp: "2026-04-16T14:11:07Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T14:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:07Z" name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v.18a6dba7ca57b2ad namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: aac48bd4-1d24-4a3c-9fef-3a1c6a2231f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v namespace: kserve-ci-e2e-test resourceVersion: "14967" uid: 88fe0cd5-1e65-4c38-b7b0-a35284232513 kind: Event lastTimestamp: "2026-04-16T14:11:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:41Z" name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v.18a6dbbb8d0abb1e namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: 42def07a-2e7a-4da9-9b43-0978f18c0f69 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5 namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: 8b6d676a-8480-44c2-9b57-2f63ffb6a9b5 kind: Event lastTimestamp: "2026-04-16T14:09:51Z" message: 'Created pod: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5-4ms9v' metadata: creationTimestamp: "2026-04-16T14:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:09:51Z" name: isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5.18a6dba1d39c4135 namespace: kserve-ci-e2e-test resourceVersion: "14968" uid: 13507167-94ac-4c28-b888-9e4ab7bc1712 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-bd7db-predictor namespace: kserve-ci-e2e-test resourceVersion: "14939" uid: af4d8ff1-8dba-4cbb-a02d-d6cc9a40684b kind: Event lastTimestamp: "2026-04-16T14:09:51Z" message: Scaled up replica set isvc-xgboost-graph-raw-bd7db-predictor-556ff547f5 from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:09:51Z" name: isvc-xgboost-graph-raw-bd7db-predictor.18a6dba1cd9c2d19 namespace: kserve-ci-e2e-test resourceVersion: "14946" uid: f46ddc2a-a451-47c3-9197-9714b266369c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:10:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-bd7db-predictor namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: a23e762f-17e9-445b-b938-450feec9b684 kind: Event lastTimestamp: "2026-04-16T14:10:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:10:21Z" name: isvc-xgboost-graph-raw-bd7db-predictor.18a6dba54c14fa4d namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: e174326d-b74c-437a-a94d-50a5960f7368 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:10:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-bd7db-predictor namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: a23e762f-17e9-445b-b938-450feec9b684 kind: Event lastTimestamp: "2026-04-16T14:10:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:10:21Z" name: isvc-xgboost-graph-raw-bd7db-predictor.18a6dba54c790698 namespace: kserve-ci-e2e-test resourceVersion: "15252" uid: 17f0686b-7e4c-4d65-b463-365951b57156 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T14:10:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-bd7db-predictor namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: a23e762f-17e9-445b-b938-450feec9b684 kind: Event lastTimestamp: "2026-04-16T14:11:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:11:21Z" name: isvc-xgboost-graph-raw-bd7db-predictor.18a6dbac49fa8a2b namespace: kserve-ci-e2e-test resourceVersion: "15711" uid: 1c2b72aa-07d8-48cb-b12f-0c75972c2488 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T14:10:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-bd7db-predictor namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: a23e762f-17e9-445b-b938-450feec9b684 kind: Event lastTimestamp: "2026-04-16T14:11:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:11:21Z" name: isvc-xgboost-graph-raw-bd7db-predictor.18a6dbac4a5f685b namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: b43acfee-f12b-430d-9fe2-62dff7c75618 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-16T14:09:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-bd7db namespace: kserve-ci-e2e-test resourceVersion: "14931" uid: 92dc0942-90ac-442a-9019-acccec4bb267 kind: Event lastTimestamp: "2026-04-16T14:11:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:11:17Z" name: isvc-xgboost-graph-raw-bd7db.18a6dba1cd652061 namespace: kserve-ci-e2e-test resourceVersion: "15689" uid: 9c8d6cbe-162b-4d49-a8b5-5883e0508fd1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-bd7db namespace: kserve-ci-e2e-test resourceVersion: "14931" uid: 92dc0942-90ac-442a-9019-acccec4bb267 kind: Event lastTimestamp: "2026-04-16T14:09:51Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-bd7db": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-bd7db": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:09:51Z" name: isvc-xgboost-graph-raw-bd7db.18a6dba1ce436b5f namespace: kserve-ci-e2e-test resourceVersion: "14949" uid: d5973aee-8f69-4a4f-bb96-0767e0a250c9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-bd7db namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: 92dc0942-90ac-442a-9019-acccec4bb267 kind: Event lastTimestamp: "2026-04-16T14:11:17Z" message: InferenceService [isvc-xgboost-graph-raw-bd7db] is Ready metadata: creationTimestamp: "2026-04-16T14:11:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:11:17Z" name: isvc-xgboost-graph-raw-bd7db.18a6dbb602686168 namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: 9a167472-ef9f-4b00-947b-83a7bd788c30 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:11:41.503737Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw namespace: kserve-ci-e2e-test resourceVersion: "15979" uid: 559b88c7-e9d3-4da4-bb5d-fefe2d5a8e9b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw to ip-10-0-141-131.ec2.internal metadata: creationTimestamp: "2026-04-16T14:11:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:11:41Z" name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw.18a6dbbb865b7dda namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 21726c02-2016-47b8-b0b0-51b9a2792695 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-66bdb995df-m8nk5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:41Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: 559b88c7-e9d3-4da4-bb5d-fefe2d5a8e9b kind: Event lastTimestamp: "2026-04-16T14:11:41Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:11:41Z" name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw.18a6dbbba04027da namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 4894a675-9683-4ba7-9483-79999df77566 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: 559b88c7-e9d3-4da4-bb5d-fefe2d5a8e9b kind: Event lastTimestamp: "2026-04-16T14:11:41Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:16546eaa530f8a9a22c60a7ffa82a496f33cd9dfcf4989c8baef0968e8392589" already present on machine metadata: creationTimestamp: "2026-04-16T14:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:41Z" name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw.18a6dbbba1e74f22 namespace: kserve-ci-e2e-test resourceVersion: "16028" uid: b8d4d76c-017f-49c1-bfc7-5f8bfd251c12 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: 559b88c7-e9d3-4da4-bb5d-fefe2d5a8e9b kind: Event lastTimestamp: "2026-04-16T14:11:42Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:11:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:42Z" name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw.18a6dbbba5d9f03a namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 55bce45d-eae1-4a7f-b363-138f46a60f92 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: 559b88c7-e9d3-4da4-bb5d-fefe2d5a8e9b kind: Event lastTimestamp: "2026-04-16T14:11:42Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:11:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:42Z" name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw.18a6dbbba64f6d7a namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: ab325347-514d-4e5e-99d4-eecf221e1f1f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: 559b88c7-e9d3-4da4-bb5d-fefe2d5a8e9b kind: Event lastTimestamp: "2026-04-16T14:11:46Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-16T14:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:46Z" name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw.18a6dbbc9419b7e9 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 85f6191d-7bc0-4f37-a8d4-34d3ba1311c7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: 559b88c7-e9d3-4da4-bb5d-fefe2d5a8e9b kind: Event lastTimestamp: "2026-04-16T14:11:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:46Z" name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw.18a6dbbc987c11ee namespace: kserve-ci-e2e-test resourceVersion: "16084" uid: ac99b757-1363-41cc-9755-9034346ef4f4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: 559b88c7-e9d3-4da4-bb5d-fefe2d5a8e9b kind: Event lastTimestamp: "2026-04-16T14:11:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:46Z" name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw.18a6dbbc992d4810 namespace: kserve-ci-e2e-test resourceVersion: "16088" uid: 1bbfa916-b0c4-4859-8748-133b08e50451 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T14:11:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: 559b88c7-e9d3-4da4-bb5d-fefe2d5a8e9b kind: Event lastTimestamp: "2026-04-16T14:12:38Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T14:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:12:38Z" name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw.18a6dbbcd067425a namespace: kserve-ci-e2e-test resourceVersion: "16487" uid: ad4a214a-c559-4aa7-b616-18fc21f8ec02 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: 559b88c7-e9d3-4da4-bb5d-fefe2d5a8e9b kind: Event lastTimestamp: "2026-04-16T14:13:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:21Z" name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw.18a6dbd2db1ae045 namespace: kserve-ci-e2e-test resourceVersion: "16937" uid: 29b7e0b6-d824-4b20-bc75-fd525540c8b8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87 namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: ad06cbc7-df13-4b65-879e-fb1928fd7cee kind: Event lastTimestamp: "2026-04-16T14:11:41Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87-tntcw' metadata: creationTimestamp: "2026-04-16T14:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:11:41Z" name: isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87.18a6dbbb85b950cc namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 4dbe7f52-f77a-4793-a5a9-2131f514b4d6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-1d91c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15958" uid: 2a58e643-5b58-4f5f-b333-cc78d080bf48 kind: Event lastTimestamp: "2026-04-16T14:11:41Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-1d91c-predictor-779d987f87 from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:11:41Z" name: isvc-xgboost-graph-raw-hpa-1d91c-predictor.18a6dbbb7d844817 namespace: kserve-ci-e2e-test resourceVersion: "15968" uid: ceb874a7-9958-4a61-89ec-a654bc8b2b47 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:11:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-1d91c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 86964b35-11d5-4dea-b566-54727f25f97a kind: Event lastTimestamp: "2026-04-16T14:12:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:12:11Z" name: isvc-xgboost-graph-raw-hpa-1d91c-predictor.18a6dbbefc3f159c namespace: kserve-ci-e2e-test resourceVersion: "16286" uid: ff399e39-d2b8-465e-8e75-06ba86835cba reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:11:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-1d91c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 86964b35-11d5-4dea-b566-54727f25f97a kind: Event lastTimestamp: "2026-04-16T14:12:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:12:11Z" name: isvc-xgboost-graph-raw-hpa-1d91c-predictor.18a6dbbefca98d52 namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: 51401968-4c15-45b1-ad03-b6c58b7be820 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T14:12:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-1d91c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: 86964b35-11d5-4dea-b566-54727f25f97a kind: Event lastTimestamp: "2026-04-16T14:13:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:12:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:11Z" name: isvc-xgboost-graph-raw-hpa-1d91c-predictor.18a6dbc5fa2394c9 namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: af617b2b-05fa-439b-9948-07ab0f406701 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T14:12:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-1d91c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: 86964b35-11d5-4dea-b566-54727f25f97a kind: Event lastTimestamp: "2026-04-16T14:13:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:12:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:11Z" name: isvc-xgboost-graph-raw-hpa-1d91c-predictor.18a6dbc5fa8ca645 namespace: kserve-ci-e2e-test resourceVersion: "16765" uid: 95c2de47-55c4-46e4-8f53-2fe4ab504487 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T14:11:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-1d91c namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: e5db5f0c-b8f7-4e23-91d4-8d7e8cef84d2 kind: Event lastTimestamp: "2026-04-16T14:12:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:12:48Z" name: isvc-xgboost-graph-raw-hpa-1d91c.18a6dbbb7d7f1049 namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: 22ca0b6d-81fc-4a21-bb7b-83b2665bdc4e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-1d91c namespace: kserve-ci-e2e-test resourceVersion: "16565" uid: e5db5f0c-b8f7-4e23-91d4-8d7e8cef84d2 kind: Event lastTimestamp: "2026-04-16T14:12:48Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-1d91c] is Ready metadata: creationTimestamp: "2026-04-16T14:12:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:12:48Z" name: isvc-xgboost-graph-raw-hpa-1d91c.18a6dbcb0842620a namespace: kserve-ci-e2e-test resourceVersion: "16566" uid: 1bb39793-3361-4d44-89bf-a933193ea540 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:13:21.617984Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-7cc0c-predictor-7f8fd4fc94-zk549 namespace: kserve-ci-e2e-test resourceVersion: "16904" uid: 7a3ef8e4-3752-4d52-878d-05d2ff127b72 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-7cc0c-predictor-7f8fd4fc94-zk549 to ip-10-0-136-114.ec2.internal metadata: creationTimestamp: "2026-04-16T14:13:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:13:21Z" name: message-dumper-raw-7cc0c-predictor-7f8fd4fc94-zk549.18a6dbd2d5a18fd5 namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: 32cbe291-c59e-4198-a365-7f8afd40fe12 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-66bdb995df-m8nk5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:21Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-7cc0c-predictor-7f8fd4fc94-zk549 namespace: kserve-ci-e2e-test resourceVersion: "16916" uid: 7a3ef8e4-3752-4d52-878d-05d2ff127b72 kind: Event lastTimestamp: "2026-04-16T14:13:21Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:13:21Z" name: message-dumper-raw-7cc0c-predictor-7f8fd4fc94-zk549.18a6dbd2dd08de44 namespace: kserve-ci-e2e-test resourceVersion: "16938" uid: 3d4e5c94-7f96-4904-9618-2217e12cd45d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-7cc0c-predictor-7f8fd4fc94-zk549 namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 7a3ef8e4-3752-4d52-878d-05d2ff127b72 kind: Event lastTimestamp: "2026-04-16T14:13:21Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-16T14:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:21Z" name: message-dumper-raw-7cc0c-predictor-7f8fd4fc94-zk549.18a6dbd2de5841df namespace: kserve-ci-e2e-test resourceVersion: "16940" uid: ef4c6817-a69a-4be1-92f3-df11b1c0df3e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-114.ec2.internal source: component: kubelet host: ip-10-0-136-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-7cc0c-predictor-7f8fd4fc94-zk549 namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 7a3ef8e4-3752-4d52-878d-05d2ff127b72 kind: Event lastTimestamp: "2026-04-16T14:13:22Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.102s (1.102s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-16T14:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:22Z" name: message-dumper-raw-7cc0c-predictor-7f8fd4fc94-zk549.18a6dbd3200cdabd namespace: kserve-ci-e2e-test resourceVersion: "16954" uid: 9d2dbc65-1d72-4446-9809-fbac6d3b17c2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-114.ec2.internal source: component: kubelet host: ip-10-0-136-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-7cc0c-predictor-7f8fd4fc94-zk549 namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 7a3ef8e4-3752-4d52-878d-05d2ff127b72 kind: Event lastTimestamp: "2026-04-16T14:13:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:22Z" name: message-dumper-raw-7cc0c-predictor-7f8fd4fc94-zk549.18a6dbd324267e81 namespace: kserve-ci-e2e-test resourceVersion: "16955" uid: b07dd50e-5387-4924-bcaa-f2e369ce55bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-114.ec2.internal source: component: kubelet host: ip-10-0-136-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-7cc0c-predictor-7f8fd4fc94-zk549 namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 7a3ef8e4-3752-4d52-878d-05d2ff127b72 kind: Event lastTimestamp: "2026-04-16T14:13:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:22Z" name: message-dumper-raw-7cc0c-predictor-7f8fd4fc94-zk549.18a6dbd32481c99a namespace: kserve-ci-e2e-test resourceVersion: "16957" uid: 5d807df1-87d0-453d-ac9b-085978e7770c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-114.ec2.internal source: component: kubelet host: ip-10-0-136-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-7cc0c-predictor-7f8fd4fc94-zk549 namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 7a3ef8e4-3752-4d52-878d-05d2ff127b72 kind: Event lastTimestamp: "2026-04-16T14:14:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:14:56Z" name: message-dumper-raw-7cc0c-predictor-7f8fd4fc94-zk549.18a6dbe90a09a35a namespace: kserve-ci-e2e-test resourceVersion: "17782" uid: 8771d417-29f0-4710-bbb2-27cfa22bd5fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-114.ec2.internal source: component: kubelet host: ip-10-0-136-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-7cc0c-predictor-7f8fd4fc94 namespace: kserve-ci-e2e-test resourceVersion: "16884" uid: 52401016-209b-4317-b94d-68021277f614 kind: Event lastTimestamp: "2026-04-16T14:13:21Z" message: 'Created pod: message-dumper-raw-7cc0c-predictor-7f8fd4fc94-zk549' metadata: creationTimestamp: "2026-04-16T14:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:21Z" name: message-dumper-raw-7cc0c-predictor-7f8fd4fc94.18a6dbd2d4fbad78 namespace: kserve-ci-e2e-test resourceVersion: "16907" uid: c44bbd3b-fad3-4a94-addd-a1fdd8f3e5b1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-7cc0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16882" uid: 1e7d931d-2014-4175-ba8e-ff22c5a60f1f kind: Event lastTimestamp: "2026-04-16T14:13:21Z" message: Scaled up replica set message-dumper-raw-7cc0c-predictor-7f8fd4fc94 from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:21Z" name: message-dumper-raw-7cc0c-predictor.18a6dbd2d1f328ff namespace: kserve-ci-e2e-test resourceVersion: "16886" uid: 282d188e-6052-4c3f-83ec-2fb32c88e07b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:13:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-7cc0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16883" uid: b6b823fe-84da-4fbd-b875-af0270d3a42d kind: Event lastTimestamp: "2026-04-16T14:13:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:13:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:51Z" name: message-dumper-raw-7cc0c-predictor.18a6dbd650ab0394 namespace: kserve-ci-e2e-test resourceVersion: "17250" uid: 6e8eb6e7-7ddb-456c-86dd-38c60a5c3736 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:13:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-7cc0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16883" uid: b6b823fe-84da-4fbd-b875-af0270d3a42d kind: Event lastTimestamp: "2026-04-16T14:13:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:13:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:51Z" name: message-dumper-raw-7cc0c-predictor.18a6dbd65109d994 namespace: kserve-ci-e2e-test resourceVersion: "17251" uid: e1f6bbec-24ae-4df7-81ab-8d8ae6f8fe16 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T14:13:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-7cc0c namespace: kserve-ci-e2e-test resourceVersion: "16876" uid: d0b05ef1-2763-4a82-8554-068ef3408e73 kind: Event lastTimestamp: "2026-04-16T14:13:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:13:23Z" name: message-dumper-raw-7cc0c.18a6dbd2d1fe1a41 namespace: kserve-ci-e2e-test resourceVersion: "16976" uid: 8295f257-ba0e-4abc-ab1f-ff442eff98db reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-7cc0c namespace: kserve-ci-e2e-test resourceVersion: "16889" uid: d0b05ef1-2763-4a82-8554-068ef3408e73 kind: Event lastTimestamp: "2026-04-16T14:13:21Z" message: 'Failed to update status for InferenceService "message-dumper-raw-7cc0c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-7cc0c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:13:21Z" name: message-dumper-raw-7cc0c.18a6dbd2d5c18922 namespace: kserve-ci-e2e-test resourceVersion: "16912" uid: 4fbd332f-6c8f-471a-9772-be3946eea0ee reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-7cc0c namespace: kserve-ci-e2e-test resourceVersion: "16974" uid: d0b05ef1-2763-4a82-8554-068ef3408e73 kind: Event lastTimestamp: "2026-04-16T14:13:23Z" message: InferenceService [message-dumper-raw-7cc0c] is Ready metadata: creationTimestamp: "2026-04-16T14:13:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:13:23Z" name: message-dumper-raw-7cc0c.18a6dbd35eb73809 namespace: kserve-ci-e2e-test resourceVersion: "16975" uid: a983c59a-419a-4f72-beff-131873e5162a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:11:31.096556Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-bd7db-78d659848c-xmbs5 namespace: kserve-ci-e2e-test resourceVersion: "15792" uid: e543c7f7-1cf7-464f-b819-74fc9125e99e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-bd7db-78d659848c-xmbs5 to ip-10-0-136-114.ec2.internal metadata: creationTimestamp: "2026-04-16T14:11:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:11:31Z" name: model-chainer-raw-bd7db-78d659848c-xmbs5.18a6dbb91a0a6707 namespace: kserve-ci-e2e-test resourceVersion: "15798" uid: eb6b34aa-7087-4a80-863b-3b2bdbd79bb8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-66bdb995df-m8nk5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:31Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-bd7db-78d659848c-xmbs5 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: e543c7f7-1cf7-464f-b819-74fc9125e99e kind: Event lastTimestamp: "2026-04-16T14:11:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-bd7db-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T14:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:31Z" name: model-chainer-raw-bd7db-78d659848c-xmbs5.18a6dbb925385da5 namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: 0c2ec94c-c303-45d4-9275-2546712b430f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-114.ec2.internal source: component: kubelet host: ip-10-0-136-114.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:32Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-bd7db-78d659848c-xmbs5 namespace: kserve-ci-e2e-test resourceVersion: "15802" uid: e543c7f7-1cf7-464f-b819-74fc9125e99e kind: Event lastTimestamp: "2026-04-16T14:11:32Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:11:32Z" name: model-chainer-raw-bd7db-78d659848c-xmbs5.18a6dbb956c1aa5f namespace: kserve-ci-e2e-test resourceVersion: "15816" uid: 654d4369-389c-4282-8370-1cb64109f6ad reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-bd7db} kind: Pod name: model-chainer-raw-bd7db-78d659848c-xmbs5 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: e543c7f7-1cf7-464f-b819-74fc9125e99e kind: Event lastTimestamp: "2026-04-16T14:11:32Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:a4f7bd996e5317ce37dda0cef29ec36fa9a5ab15194a986a81c996dc01b5a316" metadata: creationTimestamp: "2026-04-16T14:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:32Z" name: model-chainer-raw-bd7db-78d659848c-xmbs5.18a6dbb9580758cb namespace: kserve-ci-e2e-test resourceVersion: "15818" uid: 337b50cf-cbf5-46aa-bc72-1d07cc9a3457 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-114.ec2.internal source: component: kubelet host: ip-10-0-136-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-bd7db} kind: Pod name: model-chainer-raw-bd7db-78d659848c-xmbs5 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: e543c7f7-1cf7-464f-b819-74fc9125e99e kind: Event lastTimestamp: "2026-04-16T14:11:35Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:a4f7bd996e5317ce37dda0cef29ec36fa9a5ab15194a986a81c996dc01b5a316" in 3.763s (3.763s including waiting). Image size: 216228434 bytes.' metadata: creationTimestamp: "2026-04-16T14:11:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:35Z" name: model-chainer-raw-bd7db-78d659848c-xmbs5.18a6dbba3853aac7 namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: 1a7a0be3-b906-4179-9d8c-296faf036ddc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-114.ec2.internal source: component: kubelet host: ip-10-0-136-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-bd7db} kind: Pod name: model-chainer-raw-bd7db-78d659848c-xmbs5 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: e543c7f7-1cf7-464f-b819-74fc9125e99e kind: Event lastTimestamp: "2026-04-16T14:11:35Z" message: 'Created container: model-chainer-raw-bd7db' metadata: creationTimestamp: "2026-04-16T14:11:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:35Z" name: model-chainer-raw-bd7db-78d659848c-xmbs5.18a6dbba3bf74716 namespace: kserve-ci-e2e-test resourceVersion: "15849" uid: b610b373-9cc1-4dba-9454-9e3ef2cb3017 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-114.ec2.internal source: component: kubelet host: ip-10-0-136-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-bd7db} kind: Pod name: model-chainer-raw-bd7db-78d659848c-xmbs5 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: e543c7f7-1cf7-464f-b819-74fc9125e99e kind: Event lastTimestamp: "2026-04-16T14:11:35Z" message: Started container model-chainer-raw-bd7db metadata: creationTimestamp: "2026-04-16T14:11:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:35Z" name: model-chainer-raw-bd7db-78d659848c-xmbs5.18a6dbba3c65c9ae namespace: kserve-ci-e2e-test resourceVersion: "15850" uid: 9bfb595b-92cd-4779-a94f-eaccfb0b2c12 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-114.ec2.internal source: component: kubelet host: ip-10-0-136-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-bd7db} kind: Pod name: model-chainer-raw-bd7db-78d659848c-xmbs5 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: e543c7f7-1cf7-464f-b819-74fc9125e99e kind: Event lastTimestamp: "2026-04-16T14:11:41Z" message: Stopping container model-chainer-raw-bd7db metadata: creationTimestamp: "2026-04-16T14:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:41Z" name: model-chainer-raw-bd7db-78d659848c-xmbs5.18a6dbbb719208d7 namespace: kserve-ci-e2e-test resourceVersion: "15913" uid: 4ed2080b-867c-4395-a841-5bb22325e29b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-114.ec2.internal source: component: kubelet host: ip-10-0-136-114.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T14:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-bd7db} kind: Pod name: model-chainer-raw-bd7db-78d659848c-xmbs5 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: e543c7f7-1cf7-464f-b819-74fc9125e99e kind: Event lastTimestamp: "2026-04-16T14:12:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T14:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:12:11Z" name: model-chainer-raw-bd7db-78d659848c-xmbs5.18a6dbbb71e7ef56 namespace: kserve-ci-e2e-test resourceVersion: "16283" uid: 07ad7a0d-10d7-4839-8e5b-dd961a4477d3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-114.ec2.internal source: component: kubelet host: ip-10-0-136-114.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-bd7db-78d659848c namespace: kserve-ci-e2e-test resourceVersion: "15789" uid: e174bbdb-44ce-46ec-837e-f629fc23e468 kind: Event lastTimestamp: "2026-04-16T14:11:31Z" message: 'Created pod: model-chainer-raw-bd7db-78d659848c-xmbs5' metadata: creationTimestamp: "2026-04-16T14:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:11:31Z" name: model-chainer-raw-bd7db-78d659848c.18a6dbb919ae3c62 namespace: kserve-ci-e2e-test resourceVersion: "15795" uid: 63715077-24ae-4daf-99d8-f8362fb3da63 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-bd7db namespace: kserve-ci-e2e-test resourceVersion: "15787" uid: 3384fdfa-126f-4b89-9639-2d4d55e7f637 kind: Event lastTimestamp: "2026-04-16T14:11:31Z" message: Scaled up replica set model-chainer-raw-bd7db-78d659848c from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:11:31Z" name: model-chainer-raw-bd7db.18a6dbb91882932c namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: 4db144c1-2070-4258-9801-929d7cb51de9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-bd7db namespace: kserve-ci-e2e-test resourceVersion: "15804" uid: ab2ca593-25e9-4ddc-9daa-c09a71d79a13 kind: Event lastTimestamp: "2026-04-16T14:11:31Z" message: InferenceGraph [model-chainer-raw-bd7db] is Ready metadata: creationTimestamp: "2026-04-16T14:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:11:31Z" name: model-chainer-raw-bd7db.18a6dbb91de06219 namespace: kserve-ci-e2e-test resourceVersion: "15806" uid: 58899aef-fea8-4d22-aba6-5e4842c7b3d0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:13:11.383182Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-1d91c-77dfd7db99-vjtfz namespace: kserve-ci-e2e-test resourceVersion: "16752" uid: b2069363-806e-4c2b-8007-cd4d5d84ddbc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-1d91c-77dfd7db99-vjtfz to ip-10-0-136-114.ec2.internal metadata: creationTimestamp: "2026-04-16T14:13:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:13:11Z" name: model-chainer-raw-hpa-1d91c-77dfd7db99-vjtfz.18a6dbd07396edcb namespace: kserve-ci-e2e-test resourceVersion: "16759" uid: 909d1fb5-e646-42d0-aa55-52dfaf4883f6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-66bdb995df-m8nk5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:11Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-1d91c-77dfd7db99-vjtfz namespace: kserve-ci-e2e-test resourceVersion: "16763" uid: b2069363-806e-4c2b-8007-cd4d5d84ddbc kind: Event lastTimestamp: "2026-04-16T14:13:11Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:13:11Z" name: model-chainer-raw-hpa-1d91c-77dfd7db99-vjtfz.18a6dbd08c9f4d36 namespace: kserve-ci-e2e-test resourceVersion: "16776" uid: 1e6866de-e14b-4675-ae1d-4cba378f372c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-1d91c} kind: Pod name: model-chainer-raw-hpa-1d91c-77dfd7db99-vjtfz namespace: kserve-ci-e2e-test resourceVersion: "16754" uid: b2069363-806e-4c2b-8007-cd4d5d84ddbc kind: Event lastTimestamp: "2026-04-16T14:13:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a4f7bd996e5317ce37dda0cef29ec36fa9a5ab15194a986a81c996dc01b5a316" already present on machine metadata: creationTimestamp: "2026-04-16T14:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:11Z" name: model-chainer-raw-hpa-1d91c-77dfd7db99-vjtfz.18a6dbd08de5cbdf namespace: kserve-ci-e2e-test resourceVersion: "16778" uid: 70dd6f65-9226-4633-8bd0-0f8850927627 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-114.ec2.internal source: component: kubelet host: ip-10-0-136-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-1d91c} kind: Pod name: model-chainer-raw-hpa-1d91c-77dfd7db99-vjtfz namespace: kserve-ci-e2e-test resourceVersion: "16754" uid: b2069363-806e-4c2b-8007-cd4d5d84ddbc kind: Event lastTimestamp: "2026-04-16T14:13:11Z" message: 'Created container: model-chainer-raw-hpa-1d91c' metadata: creationTimestamp: "2026-04-16T14:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:11Z" name: model-chainer-raw-hpa-1d91c-77dfd7db99-vjtfz.18a6dbd091ad9366 namespace: kserve-ci-e2e-test resourceVersion: "16780" uid: 0f7a3026-c25f-4038-ae1c-0523ed9c345e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-114.ec2.internal source: component: kubelet host: ip-10-0-136-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-1d91c} kind: Pod name: model-chainer-raw-hpa-1d91c-77dfd7db99-vjtfz namespace: kserve-ci-e2e-test resourceVersion: "16754" uid: b2069363-806e-4c2b-8007-cd4d5d84ddbc kind: Event lastTimestamp: "2026-04-16T14:13:11Z" message: Started container model-chainer-raw-hpa-1d91c metadata: creationTimestamp: "2026-04-16T14:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:11Z" name: model-chainer-raw-hpa-1d91c-77dfd7db99-vjtfz.18a6dbd0920cd534 namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 4142c4a9-17a2-466b-a629-e14f655e085c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-114.ec2.internal source: component: kubelet host: ip-10-0-136-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-1d91c} kind: Pod name: model-chainer-raw-hpa-1d91c-77dfd7db99-vjtfz namespace: kserve-ci-e2e-test resourceVersion: "16754" uid: b2069363-806e-4c2b-8007-cd4d5d84ddbc kind: Event lastTimestamp: "2026-04-16T14:13:21Z" message: Stopping container model-chainer-raw-hpa-1d91c metadata: creationTimestamp: "2026-04-16T14:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:21Z" name: model-chainer-raw-hpa-1d91c-77dfd7db99-vjtfz.18a6dbd2cb76a450 namespace: kserve-ci-e2e-test resourceVersion: "16871" uid: 68e5d78f-91f9-4d19-ac30-4b8d1639e78f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-114.ec2.internal source: component: kubelet host: ip-10-0-136-114.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T14:13:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-1d91c} kind: Pod name: model-chainer-raw-hpa-1d91c-77dfd7db99-vjtfz namespace: kserve-ci-e2e-test resourceVersion: "16754" uid: b2069363-806e-4c2b-8007-cd4d5d84ddbc kind: Event lastTimestamp: "2026-04-16T14:13:48Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T14:13:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:48Z" name: model-chainer-raw-hpa-1d91c-77dfd7db99-vjtfz.18a6dbd3583f4ccc namespace: kserve-ci-e2e-test resourceVersion: "17226" uid: e73a43cf-12f8-4e21-a337-092de4325402 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-114.ec2.internal source: component: kubelet host: ip-10-0-136-114.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-1d91c-77dfd7db99 namespace: kserve-ci-e2e-test resourceVersion: "16749" uid: a4b49c93-09f2-4423-9fa5-995a83c57c4c kind: Event lastTimestamp: "2026-04-16T14:13:11Z" message: 'Created pod: model-chainer-raw-hpa-1d91c-77dfd7db99-vjtfz' metadata: creationTimestamp: "2026-04-16T14:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:11Z" name: model-chainer-raw-hpa-1d91c-77dfd7db99.18a6dbd07324adac namespace: kserve-ci-e2e-test resourceVersion: "16755" uid: ec8bd9c1-db20-465f-b75e-0f15258fd35e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-1d91c namespace: kserve-ci-e2e-test resourceVersion: "16747" uid: 23f7b793-66f4-4f80-8279-a493f1f55334 kind: Event lastTimestamp: "2026-04-16T14:13:11Z" message: Scaled up replica set model-chainer-raw-hpa-1d91c-77dfd7db99 from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:11Z" name: model-chainer-raw-hpa-1d91c.18a6dbd072393896 namespace: kserve-ci-e2e-test resourceVersion: "16750" uid: 7acb8e40-e918-4b0f-b9e1-a80ff2a3d24e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-1d91c namespace: kserve-ci-e2e-test resourceVersion: "16767" uid: 2fc1263b-c03d-4c75-8c9b-554314a45520 kind: Event lastTimestamp: "2026-04-16T14:13:11Z" message: InferenceGraph [model-chainer-raw-hpa-1d91c] is Ready metadata: creationTimestamp: "2026-04-16T14:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:13:11Z" name: model-chainer-raw-hpa-1d91c.18a6dbd078878a8b namespace: kserve-ci-e2e-test resourceVersion: "16769" uid: 2ed46a2b-5f91-499f-95b2-e4a06e26077d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:19:13.635233Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-eca71-predictor-df48fb76-48pcr namespace: kserve-ci-e2e-test resourceVersion: "19938" uid: 3a0cbfcd-168e-433a-9b1b-c7d1338f9d9f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-eca71-predictor-df48fb76-48pcr to ip-10-0-141-131.ec2.internal metadata: creationTimestamp: "2026-04-16T14:19:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:19:13Z" name: raw-sklearn-eca71-predictor-df48fb76-48pcr.18a6dc24cb7e9115 namespace: kserve-ci-e2e-test resourceVersion: "19943" uid: 85005147-df75-4fb8-b123-647085670a95 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-66bdb995df-m8nk5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:14Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-eca71-predictor-df48fb76-48pcr namespace: kserve-ci-e2e-test resourceVersion: "19950" uid: 3a0cbfcd-168e-433a-9b1b-c7d1338f9d9f kind: Event lastTimestamp: "2026-04-16T14:19:14Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:19:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:19:14Z" name: raw-sklearn-eca71-predictor-df48fb76-48pcr.18a6dc24e4d4bd38 namespace: kserve-ci-e2e-test resourceVersion: "19964" uid: 0f5363e8-88a7-42b4-889e-d163d99a6a9b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-eca71-predictor-df48fb76-48pcr namespace: kserve-ci-e2e-test resourceVersion: "19941" uid: 3a0cbfcd-168e-433a-9b1b-c7d1338f9d9f kind: Event lastTimestamp: "2026-04-16T14:19:14Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:16546eaa530f8a9a22c60a7ffa82a496f33cd9dfcf4989c8baef0968e8392589" already present on machine metadata: creationTimestamp: "2026-04-16T14:19:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:14Z" name: raw-sklearn-eca71-predictor-df48fb76-48pcr.18a6dc24e60e17ac namespace: kserve-ci-e2e-test resourceVersion: "19966" uid: 4ae4cafc-ed04-40db-b5a1-8c199ae4c34b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-eca71-predictor-df48fb76-48pcr namespace: kserve-ci-e2e-test resourceVersion: "19941" uid: 3a0cbfcd-168e-433a-9b1b-c7d1338f9d9f kind: Event lastTimestamp: "2026-04-16T14:19:14Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:19:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:14Z" name: raw-sklearn-eca71-predictor-df48fb76-48pcr.18a6dc24e94d441b namespace: kserve-ci-e2e-test resourceVersion: "19967" uid: 58cbed30-85e7-49bf-84fd-8a247b2cd37c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-eca71-predictor-df48fb76-48pcr namespace: kserve-ci-e2e-test resourceVersion: "19941" uid: 3a0cbfcd-168e-433a-9b1b-c7d1338f9d9f kind: Event lastTimestamp: "2026-04-16T14:19:14Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:19:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:14Z" name: raw-sklearn-eca71-predictor-df48fb76-48pcr.18a6dc24e9b0b36a namespace: kserve-ci-e2e-test resourceVersion: "19968" uid: 2d8b09b5-fbbf-41e2-a356-04f67d33994f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-eca71-predictor-df48fb76-48pcr namespace: kserve-ci-e2e-test resourceVersion: "19941" uid: 3a0cbfcd-168e-433a-9b1b-c7d1338f9d9f kind: Event lastTimestamp: "2026-04-16T14:19:18Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1362" already present on machine metadata: creationTimestamp: "2026-04-16T14:19:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:18Z" name: raw-sklearn-eca71-predictor-df48fb76-48pcr.18a6dc25ea5f65e9 namespace: kserve-ci-e2e-test resourceVersion: "20009" uid: 7f142cab-ce10-41fb-a818-75a64a54033c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-eca71-predictor-df48fb76-48pcr namespace: kserve-ci-e2e-test resourceVersion: "19941" uid: 3a0cbfcd-168e-433a-9b1b-c7d1338f9d9f kind: Event lastTimestamp: "2026-04-16T14:19:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:19:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:18Z" name: raw-sklearn-eca71-predictor-df48fb76-48pcr.18a6dc25ed380d40 namespace: kserve-ci-e2e-test resourceVersion: "20013" uid: 79d83b0d-acd8-4ab6-8cc0-b95782c597b3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-eca71-predictor-df48fb76-48pcr namespace: kserve-ci-e2e-test resourceVersion: "19941" uid: 3a0cbfcd-168e-433a-9b1b-c7d1338f9d9f kind: Event lastTimestamp: "2026-04-16T14:19:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:19:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:18Z" name: raw-sklearn-eca71-predictor-df48fb76-48pcr.18a6dc25ed9af556 namespace: kserve-ci-e2e-test resourceVersion: "20014" uid: e82d33c3-5e66-4267-88bc-a7ffb9f02b56 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T14:19:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-eca71-predictor-df48fb76-48pcr namespace: kserve-ci-e2e-test resourceVersion: "19941" uid: 3a0cbfcd-168e-433a-9b1b-c7d1338f9d9f kind: Event lastTimestamp: "2026-04-16T14:20:22Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T14:19:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:20:22Z" name: raw-sklearn-eca71-predictor-df48fb76-48pcr.18a6dc262647aa98 namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: 0d1b7b5c-4e2a-485f-a574-f5e11187e1d0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-eca71-predictor-df48fb76-48pcr namespace: kserve-ci-e2e-test resourceVersion: "19941" uid: 3a0cbfcd-168e-433a-9b1b-c7d1338f9d9f kind: Event lastTimestamp: "2026-04-16T14:20:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:20:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:20:33Z" name: raw-sklearn-eca71-predictor-df48fb76-48pcr.18a6dc3773caece1 namespace: kserve-ci-e2e-test resourceVersion: "20583" uid: 8f93c4dc-18ba-4992-9793-f799b7fa149e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-eca71-predictor-df48fb76 namespace: kserve-ci-e2e-test resourceVersion: "19930" uid: 26616a53-3e6d-4171-9909-d36139853fa6 kind: Event lastTimestamp: "2026-04-16T14:19:13Z" message: 'Created pod: raw-sklearn-eca71-predictor-df48fb76-48pcr' metadata: creationTimestamp: "2026-04-16T14:19:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:19:13Z" name: raw-sklearn-eca71-predictor-df48fb76.18a6dc24cb139880 namespace: kserve-ci-e2e-test resourceVersion: "19940" uid: 6a3c2bf7-7d21-4b9a-ba49-4b63251724f7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-eca71-predictor namespace: kserve-ci-e2e-test resourceVersion: "19928" uid: 29cfb466-d352-41e0-9ec2-2ab5208775f4 kind: Event lastTimestamp: "2026-04-16T14:19:13Z" message: Scaled up replica set raw-sklearn-eca71-predictor-df48fb76 from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:19:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:19:13Z" name: raw-sklearn-eca71-predictor.18a6dc24c8dac8aa namespace: kserve-ci-e2e-test resourceVersion: "19932" uid: 27dc37a1-62fa-497c-b5ac-904c67d0ec94 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:19:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-eca71-predictor namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: df991d65-945f-4784-81db-aa95deef757f kind: Event lastTimestamp: "2026-04-16T14:19:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:19:43Z" name: raw-sklearn-eca71-predictor.18a6dc28476524be namespace: kserve-ci-e2e-test resourceVersion: "20193" uid: 36295e25-e23c-4ca5-a86c-f1e43b104101 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:19:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-eca71-predictor namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: df991d65-945f-4784-81db-aa95deef757f kind: Event lastTimestamp: "2026-04-16T14:19:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:19:43Z" name: raw-sklearn-eca71-predictor.18a6dc2847c33684 namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: bb1df107-c97a-4a58-a219-49a8222832cb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:19:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-eca71-predictor namespace: kserve-ci-e2e-test resourceVersion: "20095" uid: df991d65-945f-4784-81db-aa95deef757f kind: Event lastTimestamp: "2026-04-16T14:20:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:19:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:20:28Z" name: raw-sklearn-eca71-predictor.18a6dc2f45d10a4d namespace: kserve-ci-e2e-test resourceVersion: "20510" uid: 190bfa32-4af7-4ba1-993e-0263adde7ec7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:19:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-eca71-predictor namespace: kserve-ci-e2e-test resourceVersion: "20095" uid: df991d65-945f-4784-81db-aa95deef757f kind: Event lastTimestamp: "2026-04-16T14:20:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:19:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:20:28Z" name: raw-sklearn-eca71-predictor.18a6dc2f46350931 namespace: kserve-ci-e2e-test resourceVersion: "20511" uid: 891d9a71-0751-429a-a36b-e35b79ea4c99 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-04-16T14:19:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-eca71 namespace: kserve-ci-e2e-test resourceVersion: "19919" uid: bb429857-24ad-46d0-900d-7ffd23c42503 kind: Event lastTimestamp: "2026-04-16T14:20:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:19:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:20:32Z" name: raw-sklearn-eca71.18a6dc24c8b60a7b namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: fa19c6ea-f7bc-4a51-9c44-f738f72883c6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:19:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-eca71 namespace: kserve-ci-e2e-test resourceVersion: "19919" uid: bb429857-24ad-46d0-900d-7ffd23c42503 kind: Event lastTimestamp: "2026-04-16T14:19:13Z" message: 'Failed to update status for InferenceService "raw-sklearn-eca71": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-eca71": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:19:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:19:13Z" name: raw-sklearn-eca71.18a6dc24c903e937 namespace: kserve-ci-e2e-test resourceVersion: "19953" uid: f1a34705-aff8-4705-bfa6-12e28470676d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-eca71 namespace: kserve-ci-e2e-test resourceVersion: "19936" uid: bb429857-24ad-46d0-900d-7ffd23c42503 kind: Event lastTimestamp: "2026-04-16T14:19:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-eca71": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:19:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:19:13Z" name: raw-sklearn-eca71.18a6dc24ce71663c namespace: kserve-ci-e2e-test resourceVersion: "19954" uid: aa187ea8-b049-4017-95bb-eb65a772af2d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-eca71 namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: bb429857-24ad-46d0-900d-7ffd23c42503 kind: Event lastTimestamp: "2026-04-16T14:20:32Z" message: InferenceService [raw-sklearn-eca71] is Ready metadata: creationTimestamp: "2026-04-16T14:20:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:20:32Z" name: raw-sklearn-eca71.18a6dc372603b89f namespace: kserve-ci-e2e-test resourceVersion: "20543" uid: 816da2ac-5a7a-4bc9-84b9-94c85531d925 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:20:33.837506Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889 namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: 5d1fcbff-1731-4fef-a12c-01571e049a76 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889 to ip-10-0-141-131.ec2.internal metadata: creationTimestamp: "2026-04-16T14:20:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:20:33Z" name: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889.18a6dc3777ec2272 namespace: kserve-ci-e2e-test resourceVersion: "20597" uid: f92edf47-c2e6-4ae5-8558-21003dbb0911 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-66bdb995df-m8nk5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:34Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889 namespace: kserve-ci-e2e-test resourceVersion: "20602" uid: 5d1fcbff-1731-4fef-a12c-01571e049a76 kind: Event lastTimestamp: "2026-04-16T14:20:34Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:20:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:20:34Z" name: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889.18a6dc3791623f17 namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: ae46d054-de03-4815-839a-9ae3fe7ce607 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889 namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 5d1fcbff-1731-4fef-a12c-01571e049a76 kind: Event lastTimestamp: "2026-04-16T14:20:34Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:16546eaa530f8a9a22c60a7ffa82a496f33cd9dfcf4989c8baef0968e8392589" already present on machine metadata: creationTimestamp: "2026-04-16T14:20:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:20:34Z" name: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889.18a6dc3792af975e namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 7c3cb7e3-ced7-4cbb-9c25-e4d77a8b5b22 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889 namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 5d1fcbff-1731-4fef-a12c-01571e049a76 kind: Event lastTimestamp: "2026-04-16T14:20:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:20:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:20:34Z" name: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889.18a6dc37959f5758 namespace: kserve-ci-e2e-test resourceVersion: "20619" uid: 38cc6cd7-8332-49ae-9a21-359e6caa1610 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889 namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 5d1fcbff-1731-4fef-a12c-01571e049a76 kind: Event lastTimestamp: "2026-04-16T14:20:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:20:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:20:34Z" name: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889.18a6dc37961b51e9 namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 354beb92-efa1-4c6f-8fca-01f1181b6d8c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889 namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 5d1fcbff-1731-4fef-a12c-01571e049a76 kind: Event lastTimestamp: "2026-04-16T14:20:38Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1362" already present on machine metadata: creationTimestamp: "2026-04-16T14:20:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:20:38Z" name: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889.18a6dc389a6efdff namespace: kserve-ci-e2e-test resourceVersion: "20657" uid: e4cd9c66-06b1-47df-8d0b-0f40eb9eb100 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889 namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 5d1fcbff-1731-4fef-a12c-01571e049a76 kind: Event lastTimestamp: "2026-04-16T14:20:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:20:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:20:38Z" name: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889.18a6dc389dec0195 namespace: kserve-ci-e2e-test resourceVersion: "20663" uid: 8c93408b-2145-4a55-a8ab-d3b9b9bab8de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889 namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 5d1fcbff-1731-4fef-a12c-01571e049a76 kind: Event lastTimestamp: "2026-04-16T14:20:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:20:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:20:38Z" name: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889.18a6dc389e7abe97 namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: 2c5478f4-84d5-4af6-8d74-a9eb70cb8c26 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T14:20:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889 namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 5d1fcbff-1731-4fef-a12c-01571e049a76 kind: Event lastTimestamp: "2026-04-16T14:21:40Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T14:20:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:21:40Z" name: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889.18a6dc38d680016b namespace: kserve-ci-e2e-test resourceVersion: "21107" uid: b4a3830d-cae9-4380-9b86-f5c19475e775 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:21:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889 namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 5d1fcbff-1731-4fef-a12c-01571e049a76 kind: Event lastTimestamp: "2026-04-16T14:21:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:21:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:21:53Z" name: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889.18a6dc4a1fe25319 namespace: kserve-ci-e2e-test resourceVersion: "21220" uid: 26388cec-180f-4fa5-b8fa-24d57bd7462f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-131.ec2.internal source: component: kubelet host: ip-10-0-141-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-16ff5-predictor-d968b69d4 namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: a51fd74b-6f5d-4dcb-baa5-1d864beea0ff kind: Event lastTimestamp: "2026-04-16T14:20:33Z" message: 'Created pod: raw-sklearn-runtime-16ff5-predictor-d968b69d4-q7889' metadata: creationTimestamp: "2026-04-16T14:20:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:20:33Z" name: raw-sklearn-runtime-16ff5-predictor-d968b69d4.18a6dc377740e9d0 namespace: kserve-ci-e2e-test resourceVersion: "20592" uid: 9c603272-4c53-4735-9f64-e1dbdcd7da85 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-16ff5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: 7acf9be3-9a69-4d29-8ee2-0ca666a6853d kind: Event lastTimestamp: "2026-04-16T14:20:33Z" message: Scaled up replica set raw-sklearn-runtime-16ff5-predictor-d968b69d4 from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:20:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:20:33Z" name: raw-sklearn-runtime-16ff5-predictor.18a6dc3773c7c7ca namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: 81f02fcb-2217-46a3-ab12-9b15003aece2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:20:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-16ff5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20580" uid: ade92837-82ce-4ba8-bd70-f5840e63a97f kind: Event lastTimestamp: "2026-04-16T14:21:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:20:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:21:18Z" name: raw-sklearn-runtime-16ff5-predictor.18a6dc3af2d534df namespace: kserve-ci-e2e-test resourceVersion: "20946" uid: f004660c-7165-4cad-9a91-832b00ee0003 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:20:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-16ff5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20580" uid: ade92837-82ce-4ba8-bd70-f5840e63a97f kind: Event lastTimestamp: "2026-04-16T14:21:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:20:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:21:18Z" name: raw-sklearn-runtime-16ff5-predictor.18a6dc3af3375b78 namespace: kserve-ci-e2e-test resourceVersion: "20947" uid: aac76315-8565-4551-a2a6-43fd67ed945b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:21:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-16ff5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20741" uid: ade92837-82ce-4ba8-bd70-f5840e63a97f kind: Event lastTimestamp: "2026-04-16T14:21:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:21:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:21:48Z" name: raw-sklearn-runtime-16ff5-predictor.18a6dc457041104d namespace: kserve-ci-e2e-test resourceVersion: "21160" uid: 3f4d2542-1413-4c88-9146-8eb6a0aae404 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:21:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-16ff5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20741" uid: ade92837-82ce-4ba8-bd70-f5840e63a97f kind: Event lastTimestamp: "2026-04-16T14:21:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:21:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:21:48Z" name: raw-sklearn-runtime-16ff5-predictor.18a6dc4570a695ca namespace: kserve-ci-e2e-test resourceVersion: "21161" uid: 87aca50d-204e-4355-b946-40e04900878a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T14:20:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-16ff5 namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: c1ac66ed-6bb3-466b-8c06-93a7969b12ba kind: Event lastTimestamp: "2026-04-16T14:21:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:20:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:21:50Z" name: raw-sklearn-runtime-16ff5.18a6dc3773d3b84a namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: 90839e10-c3eb-4b64-91f3-eb53c9300188 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-16ff5 namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: c1ac66ed-6bb3-466b-8c06-93a7969b12ba kind: Event lastTimestamp: "2026-04-16T14:20:33Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-16ff5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-16ff5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:20:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:20:33Z" name: raw-sklearn-runtime-16ff5.18a6dc37745d5bb6 namespace: kserve-ci-e2e-test resourceVersion: "20586" uid: bec0b964-07ad-4386-afbe-41ed59464ba0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:21:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-16ff5 namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: c1ac66ed-6bb3-466b-8c06-93a7969b12ba kind: Event lastTimestamp: "2026-04-16T14:21:50Z" message: InferenceService [raw-sklearn-runtime-16ff5] is Ready metadata: creationTimestamp: "2026-04-16T14:21:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:21:50Z" name: raw-sklearn-runtime-16ff5.18a6dc4962aea434 namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: 904bd0f0-dbec-43f7-aef6-e89b26a3da25 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "21417"