--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:47:29.440341Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-bd8d89-predictor-77996d68c6-lmmml namespace: kserve-ci-e2e-test resourceVersion: "18528" uid: 2af1bab9-0468-42de-94e7-e6081a642248 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-bd8d89-predictor-77996d68c6-lmmml to ip-10-0-137-103.ec2.internal metadata: creationTimestamp: "2026-04-24T22:47:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:47:29Z" name: isvc-init-fail-bd8d89-predictor-77996d68c6-lmmml.18a96c85b197f165 namespace: kserve-ci-e2e-test resourceVersion: "18532" uid: 83cea816-ffc6-4ff3-bf6e-a393f7d7d5e7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58c7c775c7-zxnqk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:29Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-bd8d89-predictor-77996d68c6-lmmml namespace: kserve-ci-e2e-test resourceVersion: "18535" uid: 2af1bab9-0468-42de-94e7-e6081a642248 kind: Event lastTimestamp: "2026-04-24T22:47:29Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:47:29Z" name: isvc-init-fail-bd8d89-predictor-77996d68c6-lmmml.18a96c85ca726d74 namespace: kserve-ci-e2e-test resourceVersion: "18544" uid: aecf035a-ca65-4ce4-9b0c-1cc1a6b70914 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:47:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-bd8d89-predictor-77996d68c6-lmmml namespace: kserve-ci-e2e-test resourceVersion: "18529" uid: 2af1bab9-0468-42de-94e7-e6081a642248 kind: Event lastTimestamp: "2026-04-24T22:47:33Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:4ae4df75945dd799b0c11b5e37a48a5ba8230ff3507d345ea07c8897a8c39885" already present on machine metadata: creationTimestamp: "2026-04-24T22:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:33Z" name: isvc-init-fail-bd8d89-predictor-77996d68c6-lmmml.18a96c85cbe6c5f1 namespace: kserve-ci-e2e-test resourceVersion: "18593" uid: 374beb1c-d378-4895-82ff-71024436a19a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:47:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-bd8d89-predictor-77996d68c6-lmmml namespace: kserve-ci-e2e-test resourceVersion: "18529" uid: 2af1bab9-0468-42de-94e7-e6081a642248 kind: Event lastTimestamp: "2026-04-24T22:47:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T22:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:34Z" name: isvc-init-fail-bd8d89-predictor-77996d68c6-lmmml.18a96c85ce617ab2 namespace: kserve-ci-e2e-test resourceVersion: "18599" uid: c1ced651-7940-4dfc-a4f4-29eb89eadf65 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:47:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-bd8d89-predictor-77996d68c6-lmmml namespace: kserve-ci-e2e-test resourceVersion: "18529" uid: 2af1bab9-0468-42de-94e7-e6081a642248 kind: Event lastTimestamp: "2026-04-24T22:47:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T22:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:34Z" name: isvc-init-fail-bd8d89-predictor-77996d68c6-lmmml.18a96c85cf3453ed namespace: kserve-ci-e2e-test resourceVersion: "18600" uid: 2bdf0741-50b4-41e2-8047-c87eeb26416b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-bd8d89-predictor-77996d68c6-lmmml namespace: kserve-ci-e2e-test resourceVersion: "18529" uid: 2af1bab9-0468-42de-94e7-e6081a642248 kind: Event lastTimestamp: "2026-04-24T22:47:34Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-04-24T22:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:34Z" name: isvc-init-fail-bd8d89-predictor-77996d68c6-lmmml.18a96c86f99f0460 namespace: kserve-ci-e2e-test resourceVersion: "18669" uid: 8ee25082-be59-41e5-8a7b-9657cc88a6f5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-bd8d89-predictor-77996d68c6 namespace: kserve-ci-e2e-test resourceVersion: "18506" uid: 659b9b41-9838-4507-9c7c-5744227c2a1b kind: Event lastTimestamp: "2026-04-24T22:47:29Z" message: 'Created pod: isvc-init-fail-bd8d89-predictor-77996d68c6-lmmml' metadata: creationTimestamp: "2026-04-24T22:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:47:29Z" name: isvc-init-fail-bd8d89-predictor-77996d68c6.18a96c85b12ffd0b namespace: kserve-ci-e2e-test resourceVersion: "18530" uid: b041a631-ee22-4aec-8ce0-4bc047bb515f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-bd8d89-predictor namespace: kserve-ci-e2e-test resourceVersion: "18498" uid: 6a05d0b0-a12b-4848-9e0d-ae8d8bc5e1ff kind: Event lastTimestamp: "2026-04-24T22:47:29Z" message: Scaled up replica set isvc-init-fail-bd8d89-predictor-77996d68c6 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:47:29Z" name: isvc-init-fail-bd8d89-predictor.18a96c85ad63b180 namespace: kserve-ci-e2e-test resourceVersion: "18510" uid: 4cd9b0bb-913c-4e45-b836-6d848fba08bc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T22:47:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-bd8d89 namespace: kserve-ci-e2e-test resourceVersion: "18476" uid: 7629b836-9f3e-4deb-8e99-746a2b819f3c kind: Event lastTimestamp: "2026-04-24T22:47:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:47:34Z" name: isvc-init-fail-bd8d89.18a96c85ad3751ca namespace: kserve-ci-e2e-test resourceVersion: "18601" uid: 4f2ba434-89fb-4891-90b0-b4a4817e34f0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-bd8d89 namespace: kserve-ci-e2e-test resourceVersion: "18476" uid: 7629b836-9f3e-4deb-8e99-746a2b819f3c kind: Event lastTimestamp: "2026-04-24T22:47:29Z" message: 'Failed to update status for InferenceService "isvc-init-fail-bd8d89": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-bd8d89": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:47:29Z" name: isvc-init-fail-bd8d89.18a96c85ae145158 namespace: kserve-ci-e2e-test resourceVersion: "18516" uid: cdcf98de-9775-4a8a-84d6-eb72d96b2e34 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-bd8d89 namespace: kserve-ci-e2e-test resourceVersion: "18476" uid: 7629b836-9f3e-4deb-8e99-746a2b819f3c kind: Event lastTimestamp: "2026-04-24T22:47:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-bd8d89": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:47:29Z" name: isvc-init-fail-bd8d89.18a96c85ae14a5ff namespace: kserve-ci-e2e-test resourceVersion: "18520" uid: a24d8f86-ffae-48d6-85b5-adb207ba4af0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:42:19.574609Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: 528c314a-1af7-4b7b-9bf8-684f8decd693 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh to ip-10-0-137-103.ec2.internal metadata: creationTimestamp: "2026-04-24T22:42:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:42:19Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh.18a96c3d8c282210 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: 20c24996-f673-4c27-bd17-152f30ddf351 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58c7c775c7-zxnqk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:19Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: 528c314a-1af7-4b7b-9bf8-684f8decd693 kind: Event lastTimestamp: "2026-04-24T22:42:19Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:42:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:42:19Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh.18a96c3da523169d namespace: kserve-ci-e2e-test resourceVersion: "15939" uid: c9d9cb8c-1649-4597-a657-4c9a7190e3fd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 528c314a-1af7-4b7b-9bf8-684f8decd693 kind: Event lastTimestamp: "2026-04-24T22:42:20Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:4ae4df75945dd799b0c11b5e37a48a5ba8230ff3507d345ea07c8897a8c39885" already present on machine metadata: creationTimestamp: "2026-04-24T22:42:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:20Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh.18a96c3da66c3e2c namespace: kserve-ci-e2e-test resourceVersion: "15942" uid: 756b5941-09f5-4977-b537-4c48a30c70af reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 528c314a-1af7-4b7b-9bf8-684f8decd693 kind: Event lastTimestamp: "2026-04-24T22:42:20Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T22:42:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:20Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh.18a96c3daa6de7d5 namespace: kserve-ci-e2e-test resourceVersion: "15944" uid: 67e38075-035c-408c-a5ae-51c870cc9f06 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 528c314a-1af7-4b7b-9bf8-684f8decd693 kind: Event lastTimestamp: "2026-04-24T22:42:20Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T22:42:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:20Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh.18a96c3dab163d40 namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: 61a12eb0-04d6-45b6-89b6-e6545d0d5ee1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 528c314a-1af7-4b7b-9bf8-684f8decd693 kind: Event lastTimestamp: "2026-04-24T22:42:23Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:42:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:23Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh.18a96c3e90e584d7 namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: fd6ae9a7-35ae-49a4-ae07-26da0ec9c99e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 528c314a-1af7-4b7b-9bf8-684f8decd693 kind: Event lastTimestamp: "2026-04-24T22:42:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:24Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh.18a96c3e9436b4ea namespace: kserve-ci-e2e-test resourceVersion: "15986" uid: fc9fb0a9-550e-45a8-904c-b51366a336c9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 528c314a-1af7-4b7b-9bf8-684f8decd693 kind: Event lastTimestamp: "2026-04-24T22:42:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:24Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh.18a96c3e94a636fe namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 2c6b89f9-aea0-4e79-9573-9391701601f7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 528c314a-1af7-4b7b-9bf8-684f8decd693 kind: Event lastTimestamp: "2026-04-24T22:42:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:24Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh.18a96c3e94b137e8 namespace: kserve-ci-e2e-test resourceVersion: "15988" uid: 447b9d44-e2c4-45c7-ac06-1c6a608d3d34 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 528c314a-1af7-4b7b-9bf8-684f8decd693 kind: Event lastTimestamp: "2026-04-24T22:42:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:24Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh.18a96c3e97872280 namespace: kserve-ci-e2e-test resourceVersion: "15990" uid: cede4e19-8180-4041-a94c-78b871679864 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 528c314a-1af7-4b7b-9bf8-684f8decd693 kind: Event lastTimestamp: "2026-04-24T22:42:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:24Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh.18a96c3e97eeee0f namespace: kserve-ci-e2e-test resourceVersion: "15991" uid: 25a64bf6-7224-4525-be87-66b7081143d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 528c314a-1af7-4b7b-9bf8-684f8decd693 kind: Event lastTimestamp: "2026-04-24T22:42:24Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:3ccd79bac03ab2ef4e561ec5dd95857876389840b9f5d257961e6d8463f88534" already present on machine metadata: creationTimestamp: "2026-04-24T22:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:24Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh.18a96c3e97fa6d4c namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: ceb1b93c-cec5-43c4-9046-f29f003eab4a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 528c314a-1af7-4b7b-9bf8-684f8decd693 kind: Event lastTimestamp: "2026-04-24T22:42:24Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-24T22:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:24Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh.18a96c3e9a88c7be namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: b3156b01-8d30-42fa-950c-c29a6fd444c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 528c314a-1af7-4b7b-9bf8-684f8decd693 kind: Event lastTimestamp: "2026-04-24T22:42:24Z" message: Started container agent metadata: creationTimestamp: "2026-04-24T22:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:24Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh.18a96c3e9af1df37 namespace: kserve-ci-e2e-test resourceVersion: "15995" uid: 4eaa37d8-a3f3-47c1-a738-2ac442f89c8c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-24T22:42:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 528c314a-1af7-4b7b-9bf8-684f8decd693 kind: Event lastTimestamp: "2026-04-24T22:44:01Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:44:01Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh.18a96c3eccd6db70 namespace: kserve-ci-e2e-test resourceVersion: "16822" uid: 32e3b480-4e80-4d09-9511-0a9ddf6405d0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T22:42:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 528c314a-1af7-4b7b-9bf8-684f8decd693 kind: Event lastTimestamp: "2026-04-24T22:44:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T22:42:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:44:01Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh.18a96c3f089b93cd namespace: kserve-ci-e2e-test resourceVersion: "16823" uid: b6ef27c8-4d34-4f17-a6ce-b6937a76d133 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:43:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 528c314a-1af7-4b7b-9bf8-684f8decd693 kind: Event lastTimestamp: "2026-04-24T22:43:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:43:44Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh.18a96c51639de0d0 namespace: kserve-ci-e2e-test resourceVersion: "16624" uid: 9edb31e0-4a9d-4f63-8f69-24b1ec92a561 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:43:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 528c314a-1af7-4b7b-9bf8-684f8decd693 kind: Event lastTimestamp: "2026-04-24T22:43:44Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-24T22:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:43:44Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh.18a96c51639e16bc namespace: kserve-ci-e2e-test resourceVersion: "16627" uid: 737643f2-9365-44ce-8f11-e2b15eb70c54 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:43:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 528c314a-1af7-4b7b-9bf8-684f8decd693 kind: Event lastTimestamp: "2026-04-24T22:43:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:43:44Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh.18a96c51639ee671 namespace: kserve-ci-e2e-test resourceVersion: "16630" uid: 1f4f0e9d-e8ef-4b66-a04e-aea85c9fbe2d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T22:43:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 528c314a-1af7-4b7b-9bf8-684f8decd693 kind: Event lastTimestamp: "2026-04-24T22:44:01Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:44:01Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh.18a96c51e4bc6320 namespace: kserve-ci-e2e-test resourceVersion: "16821" uid: b650c604-8643-4fd3-9693-8296ecea4fa7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-d7349-predictor-7d5d69f59d namespace: kserve-ci-e2e-test resourceVersion: "15912" uid: 9e9914d3-b38d-41a3-9043-4ce6ba8d6229 kind: Event lastTimestamp: "2026-04-24T22:42:19Z" message: 'Created pod: isvc-logger-raw-d7349-predictor-7d5d69f59d-bxknh' metadata: creationTimestamp: "2026-04-24T22:42:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:42:19Z" name: isvc-logger-raw-d7349-predictor-7d5d69f59d.18a96c3d8b9c4e67 namespace: kserve-ci-e2e-test resourceVersion: "15924" uid: 5827e512-aa01-4134-8e3d-8a990eb64ff5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-d7349-predictor namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 7c15de80-3a90-4891-9f51-8d41a7aa9a93 kind: Event lastTimestamp: "2026-04-24T22:42:19Z" message: Scaled up replica set isvc-logger-raw-d7349-predictor-7d5d69f59d from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:42:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:42:19Z" name: isvc-logger-raw-d7349-predictor.18a96c3d88714aeb namespace: kserve-ci-e2e-test resourceVersion: "15914" uid: 862895ce-9582-4298-93b4-696b7814aaa7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:42:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-d7349-predictor namespace: kserve-ci-e2e-test resourceVersion: "15911" uid: 5c9566e0-7c3e-427f-8300-8afa6991428d kind: Event lastTimestamp: "2026-04-24T22:43:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:42:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:43:04Z" name: isvc-logger-raw-d7349-predictor.18a96c41070649d5 namespace: kserve-ci-e2e-test resourceVersion: "16304" uid: 574b10bb-7240-47af-9a31-bec8c3373198 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:42:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-d7349-predictor namespace: kserve-ci-e2e-test resourceVersion: "15911" uid: 5c9566e0-7c3e-427f-8300-8afa6991428d kind: Event lastTimestamp: "2026-04-24T22:43:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:42:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:43:04Z" name: isvc-logger-raw-d7349-predictor.18a96c41076cba62 namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: 21351d56-8067-41bc-b224-b27affb9f243 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:43:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-d7349-predictor namespace: kserve-ci-e2e-test resourceVersion: "16084" uid: 5c9566e0-7c3e-427f-8300-8afa6991428d kind: Event lastTimestamp: "2026-04-24T22:43:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:43:34Z" name: isvc-logger-raw-d7349-predictor.18a96c4b83ed9d7a namespace: kserve-ci-e2e-test resourceVersion: "16521" uid: 34462803-308c-4380-a8bd-70467b7b6f25 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:43:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-d7349-predictor namespace: kserve-ci-e2e-test resourceVersion: "16084" uid: 5c9566e0-7c3e-427f-8300-8afa6991428d kind: Event lastTimestamp: "2026-04-24T22:43:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:43:34Z" name: isvc-logger-raw-d7349-predictor.18a96c4b846de610 namespace: kserve-ci-e2e-test resourceVersion: "16522" uid: 7ff42f56-c3e3-4697-89aa-95f3fc6966c1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T22:42:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-d7349 namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: f6ec793e-36f1-4764-812c-ee91a1a6b8f7 kind: Event lastTimestamp: "2026-04-24T22:43:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:42:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:43:32Z" name: isvc-logger-raw-d7349.18a96c3d884f51da namespace: kserve-ci-e2e-test resourceVersion: "16503" uid: fe7c7119-3c18-4dff-a679-f85b4d686537 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-d7349 namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: f6ec793e-36f1-4764-812c-ee91a1a6b8f7 kind: Event lastTimestamp: "2026-04-24T22:42:19Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-d7349": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-d7349": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:42:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:42:19Z" name: isvc-logger-raw-d7349.18a96c3d88c727e2 namespace: kserve-ci-e2e-test resourceVersion: "15916" uid: fc403c52-0e85-4c83-b90b-4834a57bdaf5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:43:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-d7349 namespace: kserve-ci-e2e-test resourceVersion: "16501" uid: f6ec793e-36f1-4764-812c-ee91a1a6b8f7 kind: Event lastTimestamp: "2026-04-24T22:43:32Z" message: InferenceService [isvc-logger-raw-d7349] is Ready metadata: creationTimestamp: "2026-04-24T22:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:43:32Z" name: isvc-logger-raw-d7349.18a96c4e6e5e4872 namespace: kserve-ci-e2e-test resourceVersion: "16502" uid: 99dc35c5-3f26-4498-ad70-b019688cfd21 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:45:55.118458Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-51530a-predictor-77f766b669-jfmsh namespace: kserve-ci-e2e-test resourceVersion: "17678" uid: ce65cd98-c037-43cf-8e8d-4a5027764ab6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-51530a-predictor-77f766b669-jfmsh to ip-10-0-137-103.ec2.internal metadata: creationTimestamp: "2026-04-24T22:45:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:45:55Z" name: isvc-primary-51530a-predictor-77f766b669-jfmsh.18a96c6fbb926290 namespace: kserve-ci-e2e-test resourceVersion: "17682" uid: 18c535d6-19e3-4afb-acd4-8a0e1e703d0d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58c7c775c7-zxnqk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:45:55Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-51530a-predictor-77f766b669-jfmsh namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: ce65cd98-c037-43cf-8e8d-4a5027764ab6 kind: Event lastTimestamp: "2026-04-24T22:45:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-primary-51530a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T22:45:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:45:55Z" name: isvc-primary-51530a-predictor-77f766b669-jfmsh.18a96c6fca792e55 namespace: kserve-ci-e2e-test resourceVersion: "17691" uid: d0385a2c-7f82-4358-aed8-e01dfcc1353d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:45:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-51530a-predictor-77f766b669-jfmsh namespace: kserve-ci-e2e-test resourceVersion: "17687" uid: ce65cd98-c037-43cf-8e8d-4a5027764ab6 kind: Event lastTimestamp: "2026-04-24T22:45:56Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:45:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:45:56Z" name: isvc-primary-51530a-predictor-77f766b669-jfmsh.18a96c6ff7fc8614 namespace: kserve-ci-e2e-test resourceVersion: "17701" uid: 7aef94b8-bab1-4f11-825f-98c9af47abdf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:45:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-51530a-predictor-77f766b669-jfmsh namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: ce65cd98-c037-43cf-8e8d-4a5027764ab6 kind: Event lastTimestamp: "2026-04-24T22:45:56Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:4ae4df75945dd799b0c11b5e37a48a5ba8230ff3507d345ea07c8897a8c39885" already present on machine metadata: creationTimestamp: "2026-04-24T22:45:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:45:56Z" name: isvc-primary-51530a-predictor-77f766b669-jfmsh.18a96c6ff94ff170 namespace: kserve-ci-e2e-test resourceVersion: "17703" uid: ec4c39b2-b6db-4227-a46f-d6ef7fd84259 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:45:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-51530a-predictor-77f766b669-jfmsh namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: ce65cd98-c037-43cf-8e8d-4a5027764ab6 kind: Event lastTimestamp: "2026-04-24T22:45:56Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T22:45:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:45:56Z" name: isvc-primary-51530a-predictor-77f766b669-jfmsh.18a96c6ffc2a3ea9 namespace: kserve-ci-e2e-test resourceVersion: "17704" uid: cdf10de5-ac99-4cf2-bcba-64e679809c4a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:45:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-51530a-predictor-77f766b669-jfmsh namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: ce65cd98-c037-43cf-8e8d-4a5027764ab6 kind: Event lastTimestamp: "2026-04-24T22:45:56Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T22:45:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:45:56Z" name: isvc-primary-51530a-predictor-77f766b669-jfmsh.18a96c6ffcce9844 namespace: kserve-ci-e2e-test resourceVersion: "17705" uid: d4e9b4a1-a004-4763-b7e2-98cf2eee5071 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:46:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-51530a-predictor-77f766b669-jfmsh namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: ce65cd98-c037-43cf-8e8d-4a5027764ab6 kind: Event lastTimestamp: "2026-04-24T22:46:00Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:46:00Z" name: isvc-primary-51530a-predictor-77f766b669-jfmsh.18a96c71044f24f9 namespace: kserve-ci-e2e-test resourceVersion: "17743" uid: a93b62b0-a671-43c7-acb1-fe280403cc9e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:46:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-51530a-predictor-77f766b669-jfmsh namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: ce65cd98-c037-43cf-8e8d-4a5027764ab6 kind: Event lastTimestamp: "2026-04-24T22:46:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:46:00Z" name: isvc-primary-51530a-predictor-77f766b669-jfmsh.18a96c71078636fd namespace: kserve-ci-e2e-test resourceVersion: "17745" uid: a8d02129-4310-41eb-bbeb-4d5026f762a6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:46:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-51530a-predictor-77f766b669-jfmsh namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: ce65cd98-c037-43cf-8e8d-4a5027764ab6 kind: Event lastTimestamp: "2026-04-24T22:46:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:46:00Z" name: isvc-primary-51530a-predictor-77f766b669-jfmsh.18a96c7107fb15ad namespace: kserve-ci-e2e-test resourceVersion: "17746" uid: a7775272-863c-4b19-8bb2-c8b26e049de8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:46:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-51530a-predictor-77f766b669-jfmsh namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: ce65cd98-c037-43cf-8e8d-4a5027764ab6 kind: Event lastTimestamp: "2026-04-24T22:46:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:46:00Z" name: isvc-primary-51530a-predictor-77f766b669-jfmsh.18a96c710805664f namespace: kserve-ci-e2e-test resourceVersion: "17749" uid: 864d2f7b-703b-45a1-9ac7-1a6f92b9347a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:46:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-51530a-predictor-77f766b669-jfmsh namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: ce65cd98-c037-43cf-8e8d-4a5027764ab6 kind: Event lastTimestamp: "2026-04-24T22:46:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:46:00Z" name: isvc-primary-51530a-predictor-77f766b669-jfmsh.18a96c710ac36527 namespace: kserve-ci-e2e-test resourceVersion: "17750" uid: 80f08c1f-5064-4d9c-9299-1d8a9954472f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:46:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-51530a-predictor-77f766b669-jfmsh namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: ce65cd98-c037-43cf-8e8d-4a5027764ab6 kind: Event lastTimestamp: "2026-04-24T22:46:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:46:00Z" name: isvc-primary-51530a-predictor-77f766b669-jfmsh.18a96c710b31809e namespace: kserve-ci-e2e-test resourceVersion: "17751" uid: eb67a3dc-5086-4eb6-bd45-7857a90a02c9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T22:46:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-51530a-predictor-77f766b669-jfmsh namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: ce65cd98-c037-43cf-8e8d-4a5027764ab6 kind: Event lastTimestamp: "2026-04-24T22:46:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:46:57Z" name: isvc-primary-51530a-predictor-77f766b669-jfmsh.18a96c714046feb5 namespace: kserve-ci-e2e-test resourceVersion: "18162" uid: 3de1407f-5696-4845-850f-b70fb1c60161 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-51530a-predictor-77f766b669-jfmsh namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: ce65cd98-c037-43cf-8e8d-4a5027764ab6 kind: Event lastTimestamp: "2026-04-24T22:47:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:29Z" name: isvc-primary-51530a-predictor-77f766b669-jfmsh.18a96c85af9b1d2b namespace: kserve-ci-e2e-test resourceVersion: "18524" uid: df115f9c-f576-44ab-96c2-ada583caa2cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-51530a-predictor-77f766b669-jfmsh namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: ce65cd98-c037-43cf-8e8d-4a5027764ab6 kind: Event lastTimestamp: "2026-04-24T22:47:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:29Z" name: isvc-primary-51530a-predictor-77f766b669-jfmsh.18a96c85af9bfbb0 namespace: kserve-ci-e2e-test resourceVersion: "18525" uid: 1ce2fd29-8288-4434-9f33-ab98d2d9a09b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-51530a-predictor-77f766b669-jfmsh namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: ce65cd98-c037-43cf-8e8d-4a5027764ab6 kind: Event lastTimestamp: "2026-04-24T22:47:32Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:47:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:32Z" name: isvc-primary-51530a-predictor-77f766b669-jfmsh.18a96c86707e3e09 namespace: kserve-ci-e2e-test resourceVersion: "18578" uid: 6b1cd944-ac0d-43a5-a7bc-e2112bb57abd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:45:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-51530a-predictor-77f766b669 namespace: kserve-ci-e2e-test resourceVersion: "17669" uid: 7971a97d-de1e-47a6-803b-9fed2c627615 kind: Event lastTimestamp: "2026-04-24T22:45:55Z" message: 'Created pod: isvc-primary-51530a-predictor-77f766b669-jfmsh' metadata: creationTimestamp: "2026-04-24T22:45:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:45:55Z" name: isvc-primary-51530a-predictor-77f766b669.18a96c6fbb24713b namespace: kserve-ci-e2e-test resourceVersion: "17679" uid: 89715472-30bc-4746-8b84-a278a870c5e5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:45:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-51530a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17665" uid: 7c55f544-e41b-4a49-aa50-098a303b72fb kind: Event lastTimestamp: "2026-04-24T22:45:55Z" message: Scaled up replica set isvc-primary-51530a-predictor-77f766b669 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:45:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:45:55Z" name: isvc-primary-51530a-predictor.18a96c6fb6658fc9 namespace: kserve-ci-e2e-test resourceVersion: "17671" uid: f0ac073f-9c7d-4247-8dfc-d2ccccd381c5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:46:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-51530a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17668" uid: 418db10f-82e0-490b-ba73-33dd623d4927 kind: Event lastTimestamp: "2026-04-24T22:46:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:46:25Z" name: isvc-primary-51530a-predictor.18a96c733569f091 namespace: kserve-ci-e2e-test resourceVersion: "17935" uid: 19960fd0-9916-477b-a775-ec60d6df5842 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:46:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-51530a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17668" uid: 418db10f-82e0-490b-ba73-33dd623d4927 kind: Event lastTimestamp: "2026-04-24T22:46:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:46:25Z" name: isvc-primary-51530a-predictor.18a96c7335dc4c18 namespace: kserve-ci-e2e-test resourceVersion: "17936" uid: 99163030-9d43-4c15-a40b-ef2d1eed68ab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T22:46:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-51530a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17827" uid: 418db10f-82e0-490b-ba73-33dd623d4927 kind: Event lastTimestamp: "2026-04-24T22:47:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:46:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:47:25Z" name: isvc-primary-51530a-predictor.18a96c7a34749253 namespace: kserve-ci-e2e-test resourceVersion: "18433" uid: ee453c3c-0fe2-43aa-b7e8-219b115e6ec7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T22:46:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-51530a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17827" uid: 418db10f-82e0-490b-ba73-33dd623d4927 kind: Event lastTimestamp: "2026-04-24T22:47:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:46:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:47:25Z" name: isvc-primary-51530a-predictor.18a96c7a34f70e50 namespace: kserve-ci-e2e-test resourceVersion: "18434" uid: 131bd04e-cb36-4459-95b1-2d018bd01b3c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-24T22:45:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-51530a namespace: kserve-ci-e2e-test resourceVersion: "17638" uid: f74250e6-fc87-42e3-9034-fa687a889ee9 kind: Event lastTimestamp: "2026-04-24T22:47:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:45:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:47:07Z" name: isvc-primary-51530a.18a96c6fb63fc828 namespace: kserve-ci-e2e-test resourceVersion: "18241" uid: b2b6a6f9-75f2-4c67-9639-8946047b0b5d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:45:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-51530a namespace: kserve-ci-e2e-test resourceVersion: "17638" uid: f74250e6-fc87-42e3-9034-fa687a889ee9 kind: Event lastTimestamp: "2026-04-24T22:45:55Z" message: 'Failed to update status for InferenceService "isvc-primary-51530a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-51530a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:45:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:45:55Z" name: isvc-primary-51530a.18a96c6fb6e74811 namespace: kserve-ci-e2e-test resourceVersion: "17673" uid: 8cc14194-cf9d-4ac5-b90e-f65f07e144b0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:45:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-51530a namespace: kserve-ci-e2e-test resourceVersion: "17638" uid: f74250e6-fc87-42e3-9034-fa687a889ee9 kind: Event lastTimestamp: "2026-04-24T22:45:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-51530a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:45:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:45:55Z" name: isvc-primary-51530a.18a96c6fb6e79621 namespace: kserve-ci-e2e-test resourceVersion: "17674" uid: b514e108-1723-4252-8197-635a1279728f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-51530a namespace: kserve-ci-e2e-test resourceVersion: "18239" uid: f74250e6-fc87-42e3-9034-fa687a889ee9 kind: Event lastTimestamp: "2026-04-24T22:47:07Z" message: InferenceService [isvc-primary-51530a] is Ready metadata: creationTimestamp: "2026-04-24T22:47:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:47:07Z" name: isvc-primary-51530a.18a96c80a4c76a10 namespace: kserve-ci-e2e-test resourceVersion: "18240" uid: 380743e3-7a39-4482-8e60-7872e3fdf1d8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:37:13.977485Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12989" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt to ip-10-0-137-103.ec2.internal metadata: creationTimestamp: "2026-04-24T22:37:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:37:13Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96bf6652614bb namespace: kserve-ci-e2e-test resourceVersion: "12994" uid: 2fa2e333-4a2d-42a3-8e97-dc07ec9d539c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58c7c775c7-zxnqk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:37:14Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "13000" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:37:14Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:37:14Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96bf67e476ab9 namespace: kserve-ci-e2e-test resourceVersion: "13009" uid: 85bc5879-f09f-47a9-917b-b54f50434476 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:37:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:37:14Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:4ae4df75945dd799b0c11b5e37a48a5ba8230ff3507d345ea07c8897a8c39885" metadata: creationTimestamp: "2026-04-24T22:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:37:14Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96bf67fdac5b6 namespace: kserve-ci-e2e-test resourceVersion: "13012" uid: fe05041c-2c1b-46f3-9cb2-4ef0c4b49d0d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:37:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:37:17Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:4ae4df75945dd799b0c11b5e37a48a5ba8230ff3507d345ea07c8897a8c39885" in 3.445s (3.445s including waiting). Image size: 301160875 bytes.' metadata: creationTimestamp: "2026-04-24T22:37:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:37:17Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96bf74d36d76b namespace: kserve-ci-e2e-test resourceVersion: "13037" uid: 8d84528d-ecf8-4f4f-a941-7ef369ee9bdd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:37:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:37:17Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T22:37:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:37:17Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96bf752235bb0 namespace: kserve-ci-e2e-test resourceVersion: "13039" uid: e74e8e33-a4b3-466c-a6b2-e5d2ea88385a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:37:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:37:17Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T22:37:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:37:17Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96bf752d9ad2a namespace: kserve-ci-e2e-test resourceVersion: "13040" uid: a97188f3-b17f-4227-89fb-7ff8f93f0bc8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:37:21Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1443" metadata: creationTimestamp: "2026-04-24T22:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:37:21Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96bf83f7f5b69 namespace: kserve-ci-e2e-test resourceVersion: "13074" uid: aeadcae7-8f14-45ee-9e8d-19245cc4ce27 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:37:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:37:34Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1443" in 12.983s (12.983s including waiting). Image size: 1561011115 bytes.' metadata: creationTimestamp: "2026-04-24T22:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:37:34Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96bfb455e12de namespace: kserve-ci-e2e-test resourceVersion: "13173" uid: e537fa08-f325-4705-8b79-f21d3f476c2a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:37:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:37:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:37:34Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96bfb48700b8b namespace: kserve-ci-e2e-test resourceVersion: "13174" uid: 0d8306d5-cc24-4db1-ac9a-112aa72d423f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:37:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:37:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:37:34Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96bfb48de3b69 namespace: kserve-ci-e2e-test resourceVersion: "13175" uid: ca65820c-a39b-434f-83d0-efa124e05d21 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:37:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:37:34Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T22:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:37:34Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96bfb48e9a696 namespace: kserve-ci-e2e-test resourceVersion: "13176" uid: 8e3213f7-02ed-47a0-b376-a61f4c56e0d5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:37:37Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.238s (2.238s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T22:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:37:37Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96bfbce569890 namespace: kserve-ci-e2e-test resourceVersion: "13191" uid: cd3359bd-9d55-432a-af33-8b77c80a1043 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:37:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:37:37Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96bfbd0f7787d namespace: kserve-ci-e2e-test resourceVersion: "13193" uid: 003a2533-4aa8-4c24-9054-8fc2c70b0be0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:37:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:37:37Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96bfbd164ba33 namespace: kserve-ci-e2e-test resourceVersion: "13194" uid: 26a26be3-2013-44ec-be59-4818fccf4f0a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:37:37Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:3ccd79bac03ab2ef4e561ec5dd95857876389840b9f5d257961e6d8463f88534" metadata: creationTimestamp: "2026-04-24T22:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:37:37Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96bfbd1702996 namespace: kserve-ci-e2e-test resourceVersion: "13195" uid: 59ebebb9-74f5-49f9-b3ce-c6d28a0dccad reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:37:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:37:39Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:3ccd79bac03ab2ef4e561ec5dd95857876389840b9f5d257961e6d8463f88534" in 2.43s (2.43s including waiting). Image size: 238035022 bytes.' metadata: creationTimestamp: "2026-04-24T22:37:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:37:39Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96bfc624dba88 namespace: kserve-ci-e2e-test resourceVersion: "13209" uid: 421c2b6f-8ce2-44af-8665-0d14714c70b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:37:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:37:39Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-24T22:37:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:37:39Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96bfc653b743d namespace: kserve-ci-e2e-test resourceVersion: "13210" uid: db9ad2a0-92a2-4253-a9db-0b080065feec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:37:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:37:39Z" message: Started container agent metadata: creationTimestamp: "2026-04-24T22:37:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:37:39Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96bfc65f05765 namespace: kserve-ci-e2e-test resourceVersion: "13212" uid: 7f7c756e-0443-4bcb-ac3b-e7a2f1b4ff3c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-24T22:37:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:39:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:03Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96bfc74a098c7 namespace: kserve-ci-e2e-test resourceVersion: "14018" uid: 53deff26-258d-4082-8ee7-d9a0daddc081 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T22:37:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:39:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T22:37:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:03Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96bfcb0659458 namespace: kserve-ci-e2e-test resourceVersion: "14019" uid: fb23873a-e451-4b36-9480-665f0604893a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:38:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:48Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96c0c7fc1de26 namespace: kserve-ci-e2e-test resourceVersion: "13810" uid: 2e7d71df-5709-45ec-8c6d-b0ee228797d0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:38:48Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-24T22:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:48Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96c0c7fc200a5 namespace: kserve-ci-e2e-test resourceVersion: "13813" uid: 5ec93658-e050-4080-8218-ae057f6ee3d0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:38:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:48Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96c0c7fc21ff7 namespace: kserve-ci-e2e-test resourceVersion: "13816" uid: 03220637-5bb7-491e-a9d9-b701aeae9a26 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T22:38:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: 5cf881dd-dc1a-4ac1-8717-4cff33fc74be kind: Event lastTimestamp: "2026-04-24T22:39:07Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:38:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:07Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt.18a96c0d3877c60e namespace: kserve-ci-e2e-test resourceVersion: "14055" uid: 421f902d-2f13-45fd-a027-ee6fc1764470 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:37:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696 namespace: kserve-ci-e2e-test resourceVersion: "12977" uid: b08ced2e-34f0-4820-8594-53ada606b2a2 kind: Event lastTimestamp: "2026-04-24T22:37:13Z" message: 'Created pod: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696-xnfjt' metadata: creationTimestamp: "2026-04-24T22:37:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:37:13Z" name: isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696.18a96bf6647d1b38 namespace: kserve-ci-e2e-test resourceVersion: "12991" uid: 39b6901c-2016-4cb8-a797-9aa41510efa4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:37:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-dd47d-predictor namespace: kserve-ci-e2e-test resourceVersion: "12976" uid: b65f171f-a550-49c7-9223-107450f2324b kind: Event lastTimestamp: "2026-04-24T22:37:13Z" message: Scaled up replica set isvc-raw-sklearn-batcher-dd47d-predictor-7bdbd56696 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:37:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:37:13Z" name: isvc-raw-sklearn-batcher-dd47d-predictor.18a96bf65bc1fec5 namespace: kserve-ci-e2e-test resourceVersion: "12980" uid: 612c4bbc-4d6a-41e1-aa2a-238b9312a90e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:37:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-dd47d-predictor namespace: kserve-ci-e2e-test resourceVersion: "12983" uid: d4443382-fd6c-4557-be3d-5b216ac2c8eb kind: Event lastTimestamp: "2026-04-24T22:38:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:37:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:38:14Z" name: isvc-raw-sklearn-batcher-dd47d-predictor.18a96bf9e0ef1fb5 namespace: kserve-ci-e2e-test resourceVersion: "13524" uid: 7039a440-727d-4912-be39-9a88c84aa6d8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:37:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-dd47d-predictor namespace: kserve-ci-e2e-test resourceVersion: "12983" uid: d4443382-fd6c-4557-be3d-5b216ac2c8eb kind: Event lastTimestamp: "2026-04-24T22:38:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:37:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:38:14Z" name: isvc-raw-sklearn-batcher-dd47d-predictor.18a96bf9e15e0653 namespace: kserve-ci-e2e-test resourceVersion: "13525" uid: 661e942f-c182-4b86-8eb7-d0aa7e0f37f3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:37:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-dd47d-predictor namespace: kserve-ci-e2e-test resourceVersion: "13127" uid: d4443382-fd6c-4557-be3d-5b216ac2c8eb kind: Event lastTimestamp: "2026-04-24T22:38:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:37:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:38:44Z" name: isvc-raw-sklearn-batcher-dd47d-predictor.18a96c00df9ddca6 namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: 7b9052c8-3db0-44a1-8686-f79ba70ff550 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:37:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-dd47d-predictor namespace: kserve-ci-e2e-test resourceVersion: "13127" uid: d4443382-fd6c-4557-be3d-5b216ac2c8eb kind: Event lastTimestamp: "2026-04-24T22:38:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:37:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:38:44Z" name: isvc-raw-sklearn-batcher-dd47d-predictor.18a96c00e0282f8b namespace: kserve-ci-e2e-test resourceVersion: "13746" uid: 77eb4358-bc9a-4f13-b3cc-340f6f380d73 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T22:37:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-dd47d namespace: kserve-ci-e2e-test resourceVersion: "12964" uid: 466062f2-7c8f-488c-9655-644050beba96 kind: Event lastTimestamp: "2026-04-24T22:38:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:37:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:38:43Z" name: isvc-raw-sklearn-batcher-dd47d.18a96bf661690822 namespace: kserve-ci-e2e-test resourceVersion: "13739" uid: 55da5d82-9bfc-4013-a97b-a5225f529c0d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:37:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-dd47d namespace: kserve-ci-e2e-test resourceVersion: "12964" uid: 466062f2-7c8f-488c-9655-644050beba96 kind: Event lastTimestamp: "2026-04-24T22:37:14Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-dd47d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-dd47d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:37:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:37:14Z" name: isvc-raw-sklearn-batcher-dd47d.18a96bf661c76217 namespace: kserve-ci-e2e-test resourceVersion: "13002" uid: 81789b14-93da-4fad-a1bd-e87f76ac7e69 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:37:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-dd47d namespace: kserve-ci-e2e-test resourceVersion: "12964" uid: 466062f2-7c8f-488c-9655-644050beba96 kind: Event lastTimestamp: "2026-04-24T22:37:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-dd47d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:37:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:37:14Z" name: isvc-raw-sklearn-batcher-dd47d.18a96bf661c7b22c namespace: kserve-ci-e2e-test resourceVersion: "13003" uid: 36336ed3-7f81-4ee6-a515-1d85838a8cc6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-dd47d namespace: kserve-ci-e2e-test resourceVersion: "13736" uid: 466062f2-7c8f-488c-9655-644050beba96 kind: Event lastTimestamp: "2026-04-24T22:38:43Z" message: InferenceService [isvc-raw-sklearn-batcher-dd47d] is Ready metadata: creationTimestamp: "2026-04-24T22:38:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:38:43Z" name: isvc-raw-sklearn-batcher-dd47d.18a96c0b26316802 namespace: kserve-ci-e2e-test resourceVersion: "13737" uid: 6ef922e0-9554-430f-8c98-48f1bcab06c8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:47:15.260090Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-51530a-predictor-687574b7c4-tjj9j namespace: kserve-ci-e2e-test resourceVersion: "18323" uid: 5cabed56-cc14-4772-b75b-1aca8bb1ca0b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-51530a-predictor-687574b7c4-tjj9j to ip-10-0-137-103.ec2.internal metadata: creationTimestamp: "2026-04-24T22:47:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:47:15Z" name: isvc-secondary-51530a-predictor-687574b7c4-tjj9j.18a96c8264628935 namespace: kserve-ci-e2e-test resourceVersion: "18327" uid: cba62cff-2b52-4aaf-8216-3b9dc6d89b12 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58c7c775c7-zxnqk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:15Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-51530a-predictor-687574b7c4-tjj9j namespace: kserve-ci-e2e-test resourceVersion: "18332" uid: 5cabed56-cc14-4772-b75b-1aca8bb1ca0b kind: Event lastTimestamp: "2026-04-24T22:47:15Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:47:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:47:15Z" name: isvc-secondary-51530a-predictor-687574b7c4-tjj9j.18a96c827d4ef292 namespace: kserve-ci-e2e-test resourceVersion: "18343" uid: ca13cb87-9fa1-4be7-88ec-b79face7fa47 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:47:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-51530a-predictor-687574b7c4-tjj9j namespace: kserve-ci-e2e-test resourceVersion: "18325" uid: 5cabed56-cc14-4772-b75b-1aca8bb1ca0b kind: Event lastTimestamp: "2026-04-24T22:47:18Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:4ae4df75945dd799b0c11b5e37a48a5ba8230ff3507d345ea07c8897a8c39885" already present on machine metadata: creationTimestamp: "2026-04-24T22:47:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:18Z" name: isvc-secondary-51530a-predictor-687574b7c4-tjj9j.18a96c827ef8345a namespace: kserve-ci-e2e-test resourceVersion: "18377" uid: 5c02f07b-dc24-4022-ab1e-63e8ed4a6d86 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:47:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-51530a-predictor-687574b7c4-tjj9j namespace: kserve-ci-e2e-test resourceVersion: "18325" uid: 5cabed56-cc14-4772-b75b-1aca8bb1ca0b kind: Event lastTimestamp: "2026-04-24T22:47:18Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T22:47:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:18Z" name: isvc-secondary-51530a-predictor-687574b7c4-tjj9j.18a96c8281fc2fc6 namespace: kserve-ci-e2e-test resourceVersion: "18380" uid: 90f3573b-65f2-4d36-bddd-8511507c90fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:47:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-51530a-predictor-687574b7c4-tjj9j namespace: kserve-ci-e2e-test resourceVersion: "18325" uid: 5cabed56-cc14-4772-b75b-1aca8bb1ca0b kind: Event lastTimestamp: "2026-04-24T22:47:18Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T22:47:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:18Z" name: isvc-secondary-51530a-predictor-687574b7c4-tjj9j.18a96c828269b6dc namespace: kserve-ci-e2e-test resourceVersion: "18383" uid: 8da38536-5c78-4e13-920f-86384a5e5fc5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-51530a-predictor-687574b7c4-tjj9j namespace: kserve-ci-e2e-test resourceVersion: "18325" uid: 5cabed56-cc14-4772-b75b-1aca8bb1ca0b kind: Event lastTimestamp: "2026-04-24T22:47:22Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-51530a-predictor-687574b7c4-tjj9j_kserve-ci-e2e-test(5cabed56-cc14-4772-b75b-1aca8bb1ca0b) metadata: creationTimestamp: "2026-04-24T22:47:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:22Z" name: isvc-secondary-51530a-predictor-687574b7c4-tjj9j.18a96c842bfc4c28 namespace: kserve-ci-e2e-test resourceVersion: "18411" uid: 97a7569e-08a7-4802-840a-4229d7ec32c3 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-51530a-predictor-687574b7c4 namespace: kserve-ci-e2e-test resourceVersion: "18313" uid: 7a6afb03-b799-46e8-be45-49452e3bf1f2 kind: Event lastTimestamp: "2026-04-24T22:47:15Z" message: 'Created pod: isvc-secondary-51530a-predictor-687574b7c4-tjj9j' metadata: creationTimestamp: "2026-04-24T22:47:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:47:15Z" name: isvc-secondary-51530a-predictor-687574b7c4.18a96c8263f36f74 namespace: kserve-ci-e2e-test resourceVersion: "18324" uid: ed535f4d-3d9a-4dc9-bd20-ce6683d11e05 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-51530a-predictor namespace: kserve-ci-e2e-test resourceVersion: "18311" uid: 07f53703-695c-4d05-8ffe-32e2b53f3480 kind: Event lastTimestamp: "2026-04-24T22:47:15Z" message: Scaled up replica set isvc-secondary-51530a-predictor-687574b7c4 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:47:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:47:15Z" name: isvc-secondary-51530a-predictor.18a96c82602fbe8e namespace: kserve-ci-e2e-test resourceVersion: "18315" uid: 22b506cb-8907-4254-aa24-30ef2739837e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-24T22:47:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-51530a namespace: kserve-ci-e2e-test resourceVersion: "18301" uid: 773df603-bb1b-430c-9b2b-6feaff4a2045 kind: Event lastTimestamp: "2026-04-24T22:47:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:47:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:47:25Z" name: isvc-secondary-51530a.18a96c82600b37d9 namespace: kserve-ci-e2e-test resourceVersion: "18437" uid: e5a4ce0c-4c95-4fa5-aba2-72a35f0bfb2f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-51530a namespace: kserve-ci-e2e-test resourceVersion: "18301" uid: 773df603-bb1b-430c-9b2b-6feaff4a2045 kind: Event lastTimestamp: "2026-04-24T22:47:15Z" message: 'Failed to update status for InferenceService "isvc-secondary-51530a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-51530a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:47:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:47:15Z" name: isvc-secondary-51530a.18a96c826073b026 namespace: kserve-ci-e2e-test resourceVersion: "18317" uid: 76dc8ce3-7a55-49af-8726-b0ee90653385 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-51530a namespace: kserve-ci-e2e-test resourceVersion: "18301" uid: 773df603-bb1b-430c-9b2b-6feaff4a2045 kind: Event lastTimestamp: "2026-04-24T22:47:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-51530a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:47:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:47:15Z" name: isvc-secondary-51530a.18a96c826073f802 namespace: kserve-ci-e2e-test resourceVersion: "18319" uid: f9f01baf-eca8-4b0f-a8c4-7cc0403ada0c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:38:49.015071Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh namespace: kserve-ci-e2e-test resourceVersion: "13825" uid: d130507f-f6d9-401d-a5e3-1328301efd06 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh to ip-10-0-137-103.ec2.internal metadata: creationTimestamp: "2026-04-24T22:38:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:38:49Z" name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh.18a96c0c85d48c27 namespace: kserve-ci-e2e-test resourceVersion: "13833" uid: 129fad69-ff4d-422c-b551-fa049f3731e8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58c7c775c7-zxnqk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:49Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh namespace: kserve-ci-e2e-test resourceVersion: "13845" uid: d130507f-f6d9-401d-a5e3-1328301efd06 kind: Event lastTimestamp: "2026-04-24T22:38:49Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:38:49Z" name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh.18a96c0c9eaae9b2 namespace: kserve-ci-e2e-test resourceVersion: "13876" uid: 8cbfbcec-e2a1-4fc8-9824-5b1add97bf0d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh namespace: kserve-ci-e2e-test resourceVersion: "13829" uid: d130507f-f6d9-401d-a5e3-1328301efd06 kind: Event lastTimestamp: "2026-04-24T22:38:49Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:4ae4df75945dd799b0c11b5e37a48a5ba8230ff3507d345ea07c8897a8c39885" already present on machine metadata: creationTimestamp: "2026-04-24T22:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:49Z" name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh.18a96c0ca02badb7 namespace: kserve-ci-e2e-test resourceVersion: "13878" uid: eafd7d9f-faf8-4675-a805-1e5fecac2741 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh namespace: kserve-ci-e2e-test resourceVersion: "13829" uid: d130507f-f6d9-401d-a5e3-1328301efd06 kind: Event lastTimestamp: "2026-04-24T22:38:49Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T22:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:49Z" name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh.18a96c0ca386f878 namespace: kserve-ci-e2e-test resourceVersion: "13880" uid: 615b6f19-8df8-4543-86e7-45183d5db2a8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh namespace: kserve-ci-e2e-test resourceVersion: "13829" uid: d130507f-f6d9-401d-a5e3-1328301efd06 kind: Event lastTimestamp: "2026-04-24T22:38:49Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T22:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:49Z" name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh.18a96c0ca400665e namespace: kserve-ci-e2e-test resourceVersion: "13881" uid: 04391070-0650-4550-8511-c3d3892b0a9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh namespace: kserve-ci-e2e-test resourceVersion: "13829" uid: d130507f-f6d9-401d-a5e3-1328301efd06 kind: Event lastTimestamp: "2026-04-24T22:38:53Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:38:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:53Z" name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh.18a96c0d81ebbb3a namespace: kserve-ci-e2e-test resourceVersion: "13930" uid: bce237be-a9ab-4800-8743-f707403a0170 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh namespace: kserve-ci-e2e-test resourceVersion: "13829" uid: d130507f-f6d9-401d-a5e3-1328301efd06 kind: Event lastTimestamp: "2026-04-24T22:38:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:38:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:53Z" name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh.18a96c0d84f24b84 namespace: kserve-ci-e2e-test resourceVersion: "13933" uid: a3150a03-62cd-410a-bbda-7a35b6a74a23 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh namespace: kserve-ci-e2e-test resourceVersion: "13829" uid: d130507f-f6d9-401d-a5e3-1328301efd06 kind: Event lastTimestamp: "2026-04-24T22:38:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:38:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:53Z" name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh.18a96c0d8564caf0 namespace: kserve-ci-e2e-test resourceVersion: "13936" uid: 8f1668a6-c067-45e9-97fa-64d1884f5e50 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh namespace: kserve-ci-e2e-test resourceVersion: "13829" uid: d130507f-f6d9-401d-a5e3-1328301efd06 kind: Event lastTimestamp: "2026-04-24T22:38:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:38:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:53Z" name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh.18a96c0d8570f076 namespace: kserve-ci-e2e-test resourceVersion: "13937" uid: 016dcac3-2d19-482a-b86f-3e71322948f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh namespace: kserve-ci-e2e-test resourceVersion: "13829" uid: d130507f-f6d9-401d-a5e3-1328301efd06 kind: Event lastTimestamp: "2026-04-24T22:38:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:38:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:53Z" name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh.18a96c0d8b790d12 namespace: kserve-ci-e2e-test resourceVersion: "13940" uid: b19f957a-7994-4e0d-8bf5-8ceeae6920df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh namespace: kserve-ci-e2e-test resourceVersion: "13829" uid: d130507f-f6d9-401d-a5e3-1328301efd06 kind: Event lastTimestamp: "2026-04-24T22:38:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:38:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:53Z" name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh.18a96c0d8be37edc namespace: kserve-ci-e2e-test resourceVersion: "13941" uid: 5019fb90-e45e-428f-8393-a184550af8d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T22:38:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh namespace: kserve-ci-e2e-test resourceVersion: "13829" uid: d130507f-f6d9-401d-a5e3-1328301efd06 kind: Event lastTimestamp: "2026-04-24T22:40:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:30Z" name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh.18a96c0dbe06fd97 namespace: kserve-ci-e2e-test resourceVersion: "14863" uid: fd5fb43c-ad20-4f78-8b21-6a128ec82e12 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh namespace: kserve-ci-e2e-test resourceVersion: "13829" uid: d130507f-f6d9-401d-a5e3-1328301efd06 kind: Event lastTimestamp: "2026-04-24T22:40:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:29Z" name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh.18a96c23db8c3c39 namespace: kserve-ci-e2e-test resourceVersion: "14778" uid: 0cbdd937-b2da-4c4a-8a40-c2727e55cf58 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh namespace: kserve-ci-e2e-test resourceVersion: "13829" uid: d130507f-f6d9-401d-a5e3-1328301efd06 kind: Event lastTimestamp: "2026-04-24T22:40:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:29Z" name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh.18a96c23db8cfc56 namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: 9f07445b-b8c7-401d-974e-1670fc03c461 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh namespace: kserve-ci-e2e-test resourceVersion: "13829" uid: d130507f-f6d9-401d-a5e3-1328301efd06 kind: Event lastTimestamp: "2026-04-24T22:40:30Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:30Z" name: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh.18a96c241846110c namespace: kserve-ci-e2e-test resourceVersion: "14862" uid: 1522371e-b46f-4eea-b303-e2741ba1fee5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-d4694-predictor-776f564644 namespace: kserve-ci-e2e-test resourceVersion: "13812" uid: 40fd62cd-9738-4e30-8cb0-d389e43f6cc6 kind: Event lastTimestamp: "2026-04-24T22:38:49Z" message: 'Created pod: isvc-sklearn-graph-raw-d4694-predictor-776f564644-wm9dh' metadata: creationTimestamp: "2026-04-24T22:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:38:49Z" name: isvc-sklearn-graph-raw-d4694-predictor-776f564644.18a96c0c850576ad namespace: kserve-ci-e2e-test resourceVersion: "13828" uid: 7d655378-4609-400d-a0fd-6df97ff0da1a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-d4694-predictor namespace: kserve-ci-e2e-test resourceVersion: "13809" uid: 0da3bd8b-04bc-48b9-b784-d81a38f1486e kind: Event lastTimestamp: "2026-04-24T22:38:48Z" message: Scaled up replica set isvc-sklearn-graph-raw-d4694-predictor-776f564644 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:38:48Z" name: isvc-sklearn-graph-raw-d4694-predictor.18a96c0c80808089 namespace: kserve-ci-e2e-test resourceVersion: "13815" uid: 8155ca85-3f8f-4d94-b066-0b26f0080eaf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:39:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-d4694-predictor namespace: kserve-ci-e2e-test resourceVersion: "13811" uid: 23af9c94-962b-4374-90b7-b716894d9789 kind: Event lastTimestamp: "2026-04-24T22:39:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:39:33Z" name: isvc-sklearn-graph-raw-d4694-predictor.18a96c0fff05b5e4 namespace: kserve-ci-e2e-test resourceVersion: "14265" uid: d4ec653b-e797-4932-bc2f-3575efd052e8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:39:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-d4694-predictor namespace: kserve-ci-e2e-test resourceVersion: "13811" uid: 23af9c94-962b-4374-90b7-b716894d9789 kind: Event lastTimestamp: "2026-04-24T22:39:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:39:33Z" name: isvc-sklearn-graph-raw-d4694-predictor.18a96c0fff6d04ee namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 5345e002-eae4-4a7b-94d9-a4c4b2a634eb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:39:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-d4694-predictor namespace: kserve-ci-e2e-test resourceVersion: "14026" uid: 23af9c94-962b-4374-90b7-b716894d9789 kind: Event lastTimestamp: "2026-04-24T22:40:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:19Z" name: isvc-sklearn-graph-raw-d4694-predictor.18a96c1a7cdc91c1 namespace: kserve-ci-e2e-test resourceVersion: "14624" uid: 0a793ef4-8821-4840-a8a5-9eb6e5278687 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:39:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-d4694-predictor namespace: kserve-ci-e2e-test resourceVersion: "14026" uid: 23af9c94-962b-4374-90b7-b716894d9789 kind: Event lastTimestamp: "2026-04-24T22:40:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:19Z" name: isvc-sklearn-graph-raw-d4694-predictor.18a96c1a7d4325a2 namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: da95b3d7-4770-46e1-9dd5-ee2c9e00114b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T22:38:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-d4694 namespace: kserve-ci-e2e-test resourceVersion: "13780" uid: fa107013-3aa7-40bd-b514-a810bc3178f6 kind: Event lastTimestamp: "2026-04-24T22:40:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:40:00Z" name: isvc-sklearn-graph-raw-d4694.18a96c0c805f0d2b namespace: kserve-ci-e2e-test resourceVersion: "14463" uid: 58556e85-2b94-4cb5-b674-eee57fdbcca3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-d4694 namespace: kserve-ci-e2e-test resourceVersion: "13780" uid: fa107013-3aa7-40bd-b514-a810bc3178f6 kind: Event lastTimestamp: "2026-04-24T22:38:48Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-d4694": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-d4694": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:38:48Z" name: isvc-sklearn-graph-raw-d4694.18a96c0c8101f0a6 namespace: kserve-ci-e2e-test resourceVersion: "13818" uid: b7c0c631-5ab0-41dc-8f9e-fd56b5ebd89c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-d4694 namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: fa107013-3aa7-40bd-b514-a810bc3178f6 kind: Event lastTimestamp: "2026-04-24T22:40:00Z" message: InferenceService [isvc-sklearn-graph-raw-d4694] is Ready metadata: creationTimestamp: "2026-04-24T22:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:40:00Z" name: isvc-sklearn-graph-raw-d4694.18a96c1d215814d3 namespace: kserve-ci-e2e-test resourceVersion: "14460" uid: cb290764-be47-4cdc-a006-87d0b1e71f2a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:40:29.291205Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 92a467b0-f71f-4b28-b810-ef6cfad58340 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c to ip-10-0-137-103.ec2.internal metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:40:29Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c.18a96c23dec0fa40 namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 3993a37a-e821-4b8b-b4c7-f1b777ee1106 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58c7c775c7-zxnqk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c namespace: kserve-ci-e2e-test resourceVersion: "14804" uid: 92a467b0-f71f-4b28-b810-ef6cfad58340 kind: Event lastTimestamp: "2026-04-24T22:40:29Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:40:29Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c.18a96c23f7724593 namespace: kserve-ci-e2e-test resourceVersion: "14847" uid: 74763dd7-0f20-4256-98ac-9792d3e0174b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: 92a467b0-f71f-4b28-b810-ef6cfad58340 kind: Event lastTimestamp: "2026-04-24T22:40:29Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:4ae4df75945dd799b0c11b5e37a48a5ba8230ff3507d345ea07c8897a8c39885" already present on machine metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:29Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c.18a96c23f8bf8c55 namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: 3bc9537d-fc66-4916-9626-85694045a8e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: 92a467b0-f71f-4b28-b810-ef6cfad58340 kind: Event lastTimestamp: "2026-04-24T22:40:29Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:29Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c.18a96c23fb76ccaa namespace: kserve-ci-e2e-test resourceVersion: "14850" uid: 00883621-39f5-4359-b411-b499264bc8ca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: 92a467b0-f71f-4b28-b810-ef6cfad58340 kind: Event lastTimestamp: "2026-04-24T22:40:29Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:29Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c.18a96c23fbfb3f32 namespace: kserve-ci-e2e-test resourceVersion: "14851" uid: 69b38416-dc23-424c-ba28-8598638bb31d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: 92a467b0-f71f-4b28-b810-ef6cfad58340 kind: Event lastTimestamp: "2026-04-24T22:40:33Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:33Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c.18a96c24de1f7f94 namespace: kserve-ci-e2e-test resourceVersion: "14906" uid: 8eb48ffa-f643-4d26-aa74-707b370c42d3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: 92a467b0-f71f-4b28-b810-ef6cfad58340 kind: Event lastTimestamp: "2026-04-24T22:40:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:33Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c.18a96c24e2742f5c namespace: kserve-ci-e2e-test resourceVersion: "14912" uid: 6449cb18-a9f6-43a7-bd6a-a90d4d771697 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: 92a467b0-f71f-4b28-b810-ef6cfad58340 kind: Event lastTimestamp: "2026-04-24T22:40:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:33Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c.18a96c24e2e159e7 namespace: kserve-ci-e2e-test resourceVersion: "14913" uid: 476818bb-2914-4a8a-8090-085bf5043032 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: 92a467b0-f71f-4b28-b810-ef6cfad58340 kind: Event lastTimestamp: "2026-04-24T22:40:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:33Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c.18a96c24e2ec9992 namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: 92bd76cf-427b-414a-a4fd-7d182c18ece5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: 92a467b0-f71f-4b28-b810-ef6cfad58340 kind: Event lastTimestamp: "2026-04-24T22:40:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:33Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c.18a96c24e5da3fa9 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 59364fbe-5e09-40e9-b13a-f23ff27f790e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: 92a467b0-f71f-4b28-b810-ef6cfad58340 kind: Event lastTimestamp: "2026-04-24T22:40:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:33Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c.18a96c24e644ed4d namespace: kserve-ci-e2e-test resourceVersion: "14920" uid: ff2f6ac1-bdf3-4a15-bdeb-c17db8b321ad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T22:40:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: 92a467b0-f71f-4b28-b810-ef6cfad58340 kind: Event lastTimestamp: "2026-04-24T22:42:10Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:40:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:10Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c.18a96c251a256e71 namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: 55e1af0f-2d70-432c-b15c-6338dc8d4af3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: 92a467b0-f71f-4b28-b810-ef6cfad58340 kind: Event lastTimestamp: "2026-04-24T22:42:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:42:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:09Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c.18a96c3b3574108c namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: dd4296f4-8cff-4b4b-828f-78511c13fc84 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: 92a467b0-f71f-4b28-b810-ef6cfad58340 kind: Event lastTimestamp: "2026-04-24T22:42:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:42:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:09Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c.18a96c3b35753b10 namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: fd0da037-d33f-476f-a27f-913b405f468f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: 92a467b0-f71f-4b28-b810-ef6cfad58340 kind: Event lastTimestamp: "2026-04-24T22:42:10Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:42:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:10Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c.18a96c3b748084f9 namespace: kserve-ci-e2e-test resourceVersion: "15806" uid: 89f204e1-0a6d-408d-be58-bd768eead7bd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4 namespace: kserve-ci-e2e-test resourceVersion: "14758" uid: 536cfc99-8afc-40c6-b894-1d815bc4a541 kind: Event lastTimestamp: "2026-04-24T22:40:29Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4-9wk9c' metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:29Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4.18a96c23de557ac4 namespace: kserve-ci-e2e-test resourceVersion: "14788" uid: fb801679-92d4-4b2b-a3e0-9242c2fd2d58 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-133b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14755" uid: 8a78d407-fb0f-4fa4-8ee2-ce64af720e31 kind: Event lastTimestamp: "2026-04-24T22:40:29Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-133b5-predictor-7766867b4 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:29Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor.18a96c23d959a0d3 namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: 4ecb4e0c-4398-4005-9512-8131565a7c73 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T22:40:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-133b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14757" uid: 72bdb2e0-d276-4ea1-87f0-0ce55a4a71f7 kind: Event lastTimestamp: "2026-04-24T22:41:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:29Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor.18a96c275880a3f0 namespace: kserve-ci-e2e-test resourceVersion: "15356" uid: beb57413-b3f4-4430-97ce-f9592943ea35 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T22:40:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-133b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14757" uid: 72bdb2e0-d276-4ea1-87f0-0ce55a4a71f7 kind: Event lastTimestamp: "2026-04-24T22:41:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:29Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor.18a96c2758f1bffd namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: b607a032-e70b-4a09-9afa-50b68c3c205f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:41:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-133b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 72bdb2e0-d276-4ea1-87f0-0ce55a4a71f7 kind: Event lastTimestamp: "2026-04-24T22:41:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:59Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor.18a96c35557d845c namespace: kserve-ci-e2e-test resourceVersion: "15606" uid: e839e673-40e5-4d4e-8fa2-50e9c4f4222d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:41:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-133b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 72bdb2e0-d276-4ea1-87f0-0ce55a4a71f7 kind: Event lastTimestamp: "2026-04-24T22:41:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:59Z" name: isvc-sklearn-graph-raw-hpa-133b5-predictor.18a96c3555ea158e namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: 079bce28-3b4d-4253-9eb8-e99752f65fe8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-133b5 namespace: kserve-ci-e2e-test resourceVersion: "14743" uid: 8c725be6-1be9-40df-9902-0aeb111d28c7 kind: Event lastTimestamp: "2026-04-24T22:41:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:41:40Z" name: isvc-sklearn-graph-raw-hpa-133b5.18a96c23d961fcaf namespace: kserve-ci-e2e-test resourceVersion: "15453" uid: 4ffa187d-1e13-4f06-b3d3-45d9122e46b4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-133b5 namespace: kserve-ci-e2e-test resourceVersion: "14743" uid: 8c725be6-1be9-40df-9902-0aeb111d28c7 kind: Event lastTimestamp: "2026-04-24T22:40:29Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-133b5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-133b5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:40:29Z" name: isvc-sklearn-graph-raw-hpa-133b5.18a96c23da695b87 namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: ed5ee99a-3650-4ded-894c-a4df8be9ece0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-133b5 namespace: kserve-ci-e2e-test resourceVersion: "15451" uid: 8c725be6-1be9-40df-9902-0aeb111d28c7 kind: Event lastTimestamp: "2026-04-24T22:41:40Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-133b5] is Ready metadata: creationTimestamp: "2026-04-24T22:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:41:40Z" name: isvc-sklearn-graph-raw-hpa-133b5.18a96c347d38b63c namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: a032c6c0-2ed5-43d5-baf1-3d7abd647282 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:43:44.865868Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2 namespace: kserve-ci-e2e-test resourceVersion: "16637" uid: b07a71e7-a076-457d-a1af-42e821516004 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2 to ip-10-0-137-103.ec2.internal metadata: creationTimestamp: "2026-04-24T22:43:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:43:44Z" name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2.18a96c5167e97b8c namespace: kserve-ci-e2e-test resourceVersion: "16641" uid: 2d50bf0a-de5e-41a3-9c0e-deac8d6f15a3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58c7c775c7-zxnqk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:43:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2 namespace: kserve-ci-e2e-test resourceVersion: "16645" uid: b07a71e7-a076-457d-a1af-42e821516004 kind: Event lastTimestamp: "2026-04-24T22:43:45Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:43:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:43:45Z" name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2.18a96c5180e64b72 namespace: kserve-ci-e2e-test resourceVersion: "16669" uid: a1c2e7e2-4ea6-466d-95e7-03570ca41989 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:43:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: b07a71e7-a076-457d-a1af-42e821516004 kind: Event lastTimestamp: "2026-04-24T22:43:45Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:4ae4df75945dd799b0c11b5e37a48a5ba8230ff3507d345ea07c8897a8c39885" already present on machine metadata: creationTimestamp: "2026-04-24T22:43:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:43:45Z" name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2.18a96c518245ac40 namespace: kserve-ci-e2e-test resourceVersion: "16671" uid: 87e19218-a0cb-4e61-92d0-b7b376841c59 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:43:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: b07a71e7-a076-457d-a1af-42e821516004 kind: Event lastTimestamp: "2026-04-24T22:43:45Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T22:43:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:43:45Z" name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2.18a96c51854f9c7e namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: 068076a5-c996-4e90-8218-9b95d5d61af8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:43:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: b07a71e7-a076-457d-a1af-42e821516004 kind: Event lastTimestamp: "2026-04-24T22:43:45Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T22:43:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:43:45Z" name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2.18a96c5185baaaec namespace: kserve-ci-e2e-test resourceVersion: "16673" uid: 47f36e75-6bbe-4973-ba76-76371d53f1f7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:43:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: b07a71e7-a076-457d-a1af-42e821516004 kind: Event lastTimestamp: "2026-04-24T22:43:49Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:43:49Z" name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2.18a96c526c176acf namespace: kserve-ci-e2e-test resourceVersion: "16710" uid: e95cc176-3e38-4687-ae70-6bcebc834d4b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:43:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: b07a71e7-a076-457d-a1af-42e821516004 kind: Event lastTimestamp: "2026-04-24T22:43:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:43:49Z" name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2.18a96c526f3bc252 namespace: kserve-ci-e2e-test resourceVersion: "16713" uid: 42f3e1a8-e950-40c6-a636-44eeb6c19f33 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:43:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: b07a71e7-a076-457d-a1af-42e821516004 kind: Event lastTimestamp: "2026-04-24T22:43:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:43:49Z" name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2.18a96c526faffb39 namespace: kserve-ci-e2e-test resourceVersion: "16714" uid: 8fddb097-6b09-4b6e-8d5f-9bf72444c92c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:43:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: b07a71e7-a076-457d-a1af-42e821516004 kind: Event lastTimestamp: "2026-04-24T22:43:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:43:49Z" name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2.18a96c526fbbdebf namespace: kserve-ci-e2e-test resourceVersion: "16715" uid: 8ffd2055-fa60-4649-9f89-23e3d36273aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:43:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: b07a71e7-a076-457d-a1af-42e821516004 kind: Event lastTimestamp: "2026-04-24T22:43:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:43:49Z" name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2.18a96c52727b67ae namespace: kserve-ci-e2e-test resourceVersion: "16718" uid: b79a5480-d864-45ee-be21-a800d7b078f1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:43:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: b07a71e7-a076-457d-a1af-42e821516004 kind: Event lastTimestamp: "2026-04-24T22:43:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:43:49Z" name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2.18a96c5272e91b04 namespace: kserve-ci-e2e-test resourceVersion: "16720" uid: 919053c4-82db-4535-ad11-597d5182ac9b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-04-24T22:43:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: b07a71e7-a076-457d-a1af-42e821516004 kind: Event lastTimestamp: "2026-04-24T22:45:43Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:45:43Z" name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2.18a96c52e3d5b0a4 namespace: kserve-ci-e2e-test resourceVersion: "17545" uid: b6199736-e712-439c-85fd-c3430cf0a003 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:45:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: b07a71e7-a076-457d-a1af-42e821516004 kind: Event lastTimestamp: "2026-04-24T22:45:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:45:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:45:55Z" name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2.18a96c6fb5583310 namespace: kserve-ci-e2e-test resourceVersion: "17666" uid: d22dece0-544c-410d-83dc-2be602808fd9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:45:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: b07a71e7-a076-457d-a1af-42e821516004 kind: Event lastTimestamp: "2026-04-24T22:45:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:45:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:45:55Z" name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2.18a96c6fb559e3a9 namespace: kserve-ci-e2e-test resourceVersion: "17667" uid: 466b282f-7c2c-4637-91c0-c1b55fc689f1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:45:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: b07a71e7-a076-457d-a1af-42e821516004 kind: Event lastTimestamp: "2026-04-24T22:46:02Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:45:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:46:02Z" name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2.18a96c703a3d6f2d namespace: kserve-ci-e2e-test resourceVersion: "17767" uid: 410439ec-bad1-487d-8159-acaed4475710 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:43:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6 namespace: kserve-ci-e2e-test resourceVersion: "16622" uid: 779bcee7-7498-4c32-84d4-2edda38c22fb kind: Event lastTimestamp: "2026-04-24T22:43:44Z" message: 'Created pod: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6-rzfw2' metadata: creationTimestamp: "2026-04-24T22:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:43:44Z" name: isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6.18a96c51676eab17 namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: 1ac1adb5-29ce-43b7-a492-434dae8c0ce4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:43:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-eb854-predictor namespace: kserve-ci-e2e-test resourceVersion: "16614" uid: b52608fc-29e0-455f-b768-3d956390eb99 kind: Event lastTimestamp: "2026-04-24T22:43:44Z" message: Scaled up replica set isvc-sklearn-scale-raw-eb854-predictor-78dd5bdfb6 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:43:44Z" name: isvc-sklearn-scale-raw-eb854-predictor.18a96c5163b494b6 namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 13e6782d-7916-4fb2-a2a5-e5d38af9b0a9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:43:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-eb854-predictor namespace: kserve-ci-e2e-test resourceVersion: "16620" uid: 9b21b283-0da1-4b79-bb87-72f1caaddfc2 kind: Event lastTimestamp: "2026-04-24T22:44:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:43:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:44:29Z" name: isvc-sklearn-scale-raw-eb854-predictor.18a96c54e23ba7fb namespace: kserve-ci-e2e-test resourceVersion: "17022" uid: ff457b12-5a8a-4932-a10d-d817e7772642 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:43:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-eb854-predictor namespace: kserve-ci-e2e-test resourceVersion: "16620" uid: 9b21b283-0da1-4b79-bb87-72f1caaddfc2 kind: Event lastTimestamp: "2026-04-24T22:44:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:43:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:44:29Z" name: isvc-sklearn-scale-raw-eb854-predictor.18a96c54e2e7b284 namespace: kserve-ci-e2e-test resourceVersion: "17023" uid: 297a2895-b16f-45e2-8734-484cc0a6a738 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:44:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-eb854-predictor namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 9b21b283-0da1-4b79-bb87-72f1caaddfc2 kind: Event lastTimestamp: "2026-04-24T22:45:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:44:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:45:44Z" name: isvc-sklearn-scale-raw-eb854-predictor.18a96c5f610f0643 namespace: kserve-ci-e2e-test resourceVersion: "17556" uid: f78df2c5-93e6-4ffe-9725-97b4d0673079 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:44:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-eb854-predictor namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 9b21b283-0da1-4b79-bb87-72f1caaddfc2 kind: Event lastTimestamp: "2026-04-24T22:45:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:44:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:45:44Z" name: isvc-sklearn-scale-raw-eb854-predictor.18a96c5f617d8795 namespace: kserve-ci-e2e-test resourceVersion: "17557" uid: 890877d3-d134-43cb-a543-2928efccabf9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-24T22:43:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-eb854 namespace: kserve-ci-e2e-test resourceVersion: "16595" uid: 1efa03ca-ae85-4fc1-9c2a-0550f146504e kind: Event lastTimestamp: "2026-04-24T22:45:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:45:53Z" name: isvc-sklearn-scale-raw-eb854.18a96c516374f327 namespace: kserve-ci-e2e-test resourceVersion: "17625" uid: b7b600b7-597d-4e98-8ce0-35817444e094 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:43:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-eb854 namespace: kserve-ci-e2e-test resourceVersion: "16595" uid: 1efa03ca-ae85-4fc1-9c2a-0550f146504e kind: Event lastTimestamp: "2026-04-24T22:43:44Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-eb854": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-eb854": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:43:44Z" name: isvc-sklearn-scale-raw-eb854.18a96c5163fe7be6 namespace: kserve-ci-e2e-test resourceVersion: "16629" uid: c0649635-0464-4b01-be8d-d9c026783cb6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:45:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-eb854 namespace: kserve-ci-e2e-test resourceVersion: "17623" uid: 1efa03ca-ae85-4fc1-9c2a-0550f146504e kind: Event lastTimestamp: "2026-04-24T22:45:53Z" message: InferenceService [isvc-sklearn-scale-raw-eb854] is Ready metadata: creationTimestamp: "2026-04-24T22:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:45:53Z" name: isvc-sklearn-scale-raw-eb854.18a96c6f5cbb021e namespace: kserve-ci-e2e-test resourceVersion: "17624" uid: 6c6df357-e43b-428d-bed1-1f9baaf928f4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:38:49.103901Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z namespace: kserve-ci-e2e-test resourceVersion: "13857" uid: bca9a52c-f7a2-4936-a746-fd24af9e0506 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z to ip-10-0-137-103.ec2.internal metadata: creationTimestamp: "2026-04-24T22:38:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:38:49Z" name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z.18a96c0c8b200853 namespace: kserve-ci-e2e-test resourceVersion: "13862" uid: 9075b088-586c-490f-8b4c-df7f2c8de2a7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58c7c775c7-zxnqk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:49Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z namespace: kserve-ci-e2e-test resourceVersion: "13868" uid: bca9a52c-f7a2-4936-a746-fd24af9e0506 kind: Event lastTimestamp: "2026-04-24T22:38:49Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:38:49Z" name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z.18a96c0ca463470f namespace: kserve-ci-e2e-test resourceVersion: "13882" uid: 1a1ff0ab-2960-441b-a5c5-e26f567486f4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z namespace: kserve-ci-e2e-test resourceVersion: "13859" uid: bca9a52c-f7a2-4936-a746-fd24af9e0506 kind: Event lastTimestamp: "2026-04-24T22:38:49Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:4ae4df75945dd799b0c11b5e37a48a5ba8230ff3507d345ea07c8897a8c39885" already present on machine metadata: creationTimestamp: "2026-04-24T22:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:49Z" name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z.18a96c0ca5d8dcd6 namespace: kserve-ci-e2e-test resourceVersion: "13884" uid: c09c6181-139a-492f-876e-8204a88f76bb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z namespace: kserve-ci-e2e-test resourceVersion: "13859" uid: bca9a52c-f7a2-4936-a746-fd24af9e0506 kind: Event lastTimestamp: "2026-04-24T22:38:49Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T22:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:49Z" name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z.18a96c0ca87f3983 namespace: kserve-ci-e2e-test resourceVersion: "13885" uid: ba113995-cd18-4293-ad70-e9bd0bed43f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z namespace: kserve-ci-e2e-test resourceVersion: "13859" uid: bca9a52c-f7a2-4936-a746-fd24af9e0506 kind: Event lastTimestamp: "2026-04-24T22:38:49Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T22:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:49Z" name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z.18a96c0ca8ebd39f namespace: kserve-ci-e2e-test resourceVersion: "13886" uid: 35eb159d-83b1-4386-be21-8dfe95e57978 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z namespace: kserve-ci-e2e-test resourceVersion: "13859" uid: bca9a52c-f7a2-4936-a746-fd24af9e0506 kind: Event lastTimestamp: "2026-04-24T22:38:54Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-24T22:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:54Z" name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z.18a96c0dbdebf2ff namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: 85fb3a56-2ff1-4fba-a017-250dd8b62aaf reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z namespace: kserve-ci-e2e-test resourceVersion: "13859" uid: bca9a52c-f7a2-4936-a746-fd24af9e0506 kind: Event lastTimestamp: "2026-04-24T22:39:12Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 18.682s (18.682s including waiting). Image size: 1306417402 bytes.' metadata: creationTimestamp: "2026-04-24T22:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:12Z" name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z.18a96c121775a0d8 namespace: kserve-ci-e2e-test resourceVersion: "14091" uid: f590a3c3-88a8-4681-a5db-5a6ab588a64a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z namespace: kserve-ci-e2e-test resourceVersion: "13859" uid: bca9a52c-f7a2-4936-a746-fd24af9e0506 kind: Event lastTimestamp: "2026-04-24T22:39:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:12Z" name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z.18a96c121acf6abb namespace: kserve-ci-e2e-test resourceVersion: "14094" uid: ec8570e3-16f0-495e-8dd8-3b5a77e4149a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z namespace: kserve-ci-e2e-test resourceVersion: "13859" uid: bca9a52c-f7a2-4936-a746-fd24af9e0506 kind: Event lastTimestamp: "2026-04-24T22:39:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:12Z" name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z.18a96c121b3ccdd9 namespace: kserve-ci-e2e-test resourceVersion: "14095" uid: c15a6514-e086-44b8-94a6-7cfa68383118 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z namespace: kserve-ci-e2e-test resourceVersion: "13859" uid: bca9a52c-f7a2-4936-a746-fd24af9e0506 kind: Event lastTimestamp: "2026-04-24T22:39:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:13Z" name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z.18a96c121b483e97 namespace: kserve-ci-e2e-test resourceVersion: "14096" uid: 650ad3a6-4db3-48c1-8765-e116304c5905 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z namespace: kserve-ci-e2e-test resourceVersion: "13859" uid: bca9a52c-f7a2-4936-a746-fd24af9e0506 kind: Event lastTimestamp: "2026-04-24T22:39:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:13Z" name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z.18a96c121e58f5fb namespace: kserve-ci-e2e-test resourceVersion: "14099" uid: cba56b36-bcfd-4af3-bca4-44ba88c99dd6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z namespace: kserve-ci-e2e-test resourceVersion: "13859" uid: bca9a52c-f7a2-4936-a746-fd24af9e0506 kind: Event lastTimestamp: "2026-04-24T22:39:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:13Z" name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z.18a96c121ec73c91 namespace: kserve-ci-e2e-test resourceVersion: "14100" uid: 0b469f5b-4f6d-4d34-af14-7d879ae7b7d1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T22:39:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z namespace: kserve-ci-e2e-test resourceVersion: "13859" uid: bca9a52c-f7a2-4936-a746-fd24af9e0506 kind: Event lastTimestamp: "2026-04-24T22:40:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:30Z" name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z.18a96c126a1c02f9 namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: fb2f8cca-1f4d-4fae-8c98-c01e665a3800 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z namespace: kserve-ci-e2e-test resourceVersion: "13859" uid: bca9a52c-f7a2-4936-a746-fd24af9e0506 kind: Event lastTimestamp: "2026-04-24T22:40:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:29Z" name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z.18a96c23e58a3b27 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: b9f87a81-fa60-4106-8443-c5abd135177d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z namespace: kserve-ci-e2e-test resourceVersion: "13859" uid: bca9a52c-f7a2-4936-a746-fd24af9e0506 kind: Event lastTimestamp: "2026-04-24T22:40:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:29Z" name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z.18a96c23e58a9f34 namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: 156da8b7-499e-4aa9-a65a-8d7a658972b7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z namespace: kserve-ci-e2e-test resourceVersion: "13859" uid: bca9a52c-f7a2-4936-a746-fd24af9e0506 kind: Event lastTimestamp: "2026-04-24T22:40:30Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:30Z" name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z.18a96c241c2bf57a namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: daa2e9f6-2131-41cd-ab12-431d5d9b9a4c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 72a02a2c-6594-4875-80d3-6bc2af923be7 kind: Event lastTimestamp: "2026-04-24T22:38:49Z" message: 'Created pod: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9-jtk6z' metadata: creationTimestamp: "2026-04-24T22:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:38:49Z" name: isvc-xgboost-graph-raw-d4694-predictor-99989f9d9.18a96c0c8ab75694 namespace: kserve-ci-e2e-test resourceVersion: "13860" uid: 9055584f-f062-431a-b7c3-ca8f53081ba3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-d4694-predictor namespace: kserve-ci-e2e-test resourceVersion: "13844" uid: 8b3534d5-61e2-453e-a248-7a4416c2d229 kind: Event lastTimestamp: "2026-04-24T22:38:49Z" message: Scaled up replica set isvc-xgboost-graph-raw-d4694-predictor-99989f9d9 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:38:49Z" name: isvc-xgboost-graph-raw-d4694-predictor.18a96c0c8825b929 namespace: kserve-ci-e2e-test resourceVersion: "13851" uid: 469c5e59-36fe-4570-a6d1-392be08b6dd5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T22:39:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-d4694-predictor namespace: kserve-ci-e2e-test resourceVersion: "13846" uid: c0ab17c6-1fbf-476e-9c53-e61e9b2c689a kind: Event lastTimestamp: "2026-04-24T22:39:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:39:49Z" name: isvc-xgboost-graph-raw-d4694-predictor.18a96c1006ccc86c namespace: kserve-ci-e2e-test resourceVersion: "14373" uid: 5532ab19-bb9b-4d43-85a0-feee2105aaa6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T22:39:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-d4694-predictor namespace: kserve-ci-e2e-test resourceVersion: "13846" uid: c0ab17c6-1fbf-476e-9c53-e61e9b2c689a kind: Event lastTimestamp: "2026-04-24T22:39:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:39:49Z" name: isvc-xgboost-graph-raw-d4694-predictor.18a96c100734ac60 namespace: kserve-ci-e2e-test resourceVersion: "14375" uid: 1fb03a59-a92b-4040-b5ab-c07a47ba5929 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:40:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-d4694-predictor namespace: kserve-ci-e2e-test resourceVersion: "14030" uid: c0ab17c6-1fbf-476e-9c53-e61e9b2c689a kind: Event lastTimestamp: "2026-04-24T22:40:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:40:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:19Z" name: isvc-xgboost-graph-raw-d4694-predictor.18a96c1e02798428 namespace: kserve-ci-e2e-test resourceVersion: "14636" uid: 2bad7bc0-c077-4151-a5cb-63969296c790 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:40:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-d4694-predictor namespace: kserve-ci-e2e-test resourceVersion: "14030" uid: c0ab17c6-1fbf-476e-9c53-e61e9b2c689a kind: Event lastTimestamp: "2026-04-24T22:40:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:40:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:19Z" name: isvc-xgboost-graph-raw-d4694-predictor.18a96c1e02e0b0fa namespace: kserve-ci-e2e-test resourceVersion: "14637" uid: df34ab20-b522-4d3b-bdd4-99baeac45ae3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T22:38:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-d4694 namespace: kserve-ci-e2e-test resourceVersion: "13824" uid: a12b510a-657d-4040-a469-f78053c4ba23 kind: Event lastTimestamp: "2026-04-24T22:40:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:40:10Z" name: isvc-xgboost-graph-raw-d4694.18a96c0c87f91199 namespace: kserve-ci-e2e-test resourceVersion: "14545" uid: 555ae0df-add7-44d5-a6ff-e618a7b4a258 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-d4694 namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: a12b510a-657d-4040-a469-f78053c4ba23 kind: Event lastTimestamp: "2026-04-24T22:40:10Z" message: InferenceService [isvc-xgboost-graph-raw-d4694] is Ready metadata: creationTimestamp: "2026-04-24T22:40:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:40:10Z" name: isvc-xgboost-graph-raw-d4694.18a96c1f78f55f47 namespace: kserve-ci-e2e-test resourceVersion: "14544" uid: 7bc2d142-3dd7-4a02-acdf-b8a03e434f9a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:40:29.444379Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946 namespace: kserve-ci-e2e-test resourceVersion: "14832" uid: 5cae113f-ed3f-4799-98a3-0804da8ad90f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946 to ip-10-0-137-103.ec2.internal metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:40:29Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946.18a96c23e7e2315e namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: a6533490-941d-4c3c-9451-d5dfb6622387 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58c7c775c7-zxnqk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946 namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 5cae113f-ed3f-4799-98a3-0804da8ad90f kind: Event lastTimestamp: "2026-04-24T22:40:29Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:40:29Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946.18a96c240178ebba namespace: kserve-ci-e2e-test resourceVersion: "14855" uid: f60a2594-4d27-4da5-ba53-1dd915eab047 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: 5cae113f-ed3f-4799-98a3-0804da8ad90f kind: Event lastTimestamp: "2026-04-24T22:40:29Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:4ae4df75945dd799b0c11b5e37a48a5ba8230ff3507d345ea07c8897a8c39885" already present on machine metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:29Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946.18a96c2402d6821e namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 5fe86a18-d828-423b-bf7e-05cf5ae709d6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: 5cae113f-ed3f-4799-98a3-0804da8ad90f kind: Event lastTimestamp: "2026-04-24T22:40:29Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:29Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946.18a96c24059ebf6f namespace: kserve-ci-e2e-test resourceVersion: "14858" uid: ba8f21da-7b5d-41f3-a5ec-b395a0885b6e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: 5cae113f-ed3f-4799-98a3-0804da8ad90f kind: Event lastTimestamp: "2026-04-24T22:40:29Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:29Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946.18a96c24060dd13e namespace: kserve-ci-e2e-test resourceVersion: "14859" uid: 06b60628-9e4e-43f0-b6c1-748e4e2f4e20 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: 5cae113f-ed3f-4799-98a3-0804da8ad90f kind: Event lastTimestamp: "2026-04-24T22:40:34Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-24T22:40:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:34Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946.18a96c2519fcc760 namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: a3ab42f3-0d7d-400d-9ea2-681b417b179e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: 5cae113f-ed3f-4799-98a3-0804da8ad90f kind: Event lastTimestamp: "2026-04-24T22:40:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:40:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:34Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946.18a96c251d5e1759 namespace: kserve-ci-e2e-test resourceVersion: "14934" uid: c83a82d8-0265-4c67-bd87-1223babc91a5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: 5cae113f-ed3f-4799-98a3-0804da8ad90f kind: Event lastTimestamp: "2026-04-24T22:40:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:40:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:34Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946.18a96c251dca74a3 namespace: kserve-ci-e2e-test resourceVersion: "14935" uid: 43fc9fa8-3e2d-451f-82df-5f7dade05dcc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: 5cae113f-ed3f-4799-98a3-0804da8ad90f kind: Event lastTimestamp: "2026-04-24T22:40:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:40:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:34Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946.18a96c251dd5c86c namespace: kserve-ci-e2e-test resourceVersion: "14936" uid: 797961dc-3a46-49fd-bdf1-35fbcf8f7759 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: 5cae113f-ed3f-4799-98a3-0804da8ad90f kind: Event lastTimestamp: "2026-04-24T22:40:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:40:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:34Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946.18a96c25206fa4bb namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: 85c9df2b-b69e-4f6e-8a38-6d9c254fca29 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: 5cae113f-ed3f-4799-98a3-0804da8ad90f kind: Event lastTimestamp: "2026-04-24T22:40:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:40:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:34Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946.18a96c2520e545f8 namespace: kserve-ci-e2e-test resourceVersion: "14939" uid: d7441810-964a-4c11-9af2-8ecebaa9f964 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T22:40:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: 5cae113f-ed3f-4799-98a3-0804da8ad90f kind: Event lastTimestamp: "2026-04-24T22:42:11Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:11Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946.18a96c268032b182 namespace: kserve-ci-e2e-test resourceVersion: "15816" uid: 086f7f7d-6574-416d-84b1-bc2e7ce31614 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: 5cae113f-ed3f-4799-98a3-0804da8ad90f kind: Event lastTimestamp: "2026-04-24T22:42:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:42:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:09Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946.18a96c3b31ab8b8b namespace: kserve-ci-e2e-test resourceVersion: "15743" uid: 70e08287-02c6-46be-852e-676fc464b5ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: 5cae113f-ed3f-4799-98a3-0804da8ad90f kind: Event lastTimestamp: "2026-04-24T22:42:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:42:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:09Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946.18a96c3b31ac144e namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: d477947a-bed7-42e6-afb2-bbb1d4920952 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: 5cae113f-ed3f-4799-98a3-0804da8ad90f kind: Event lastTimestamp: "2026-04-24T22:42:10Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:42:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:10Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946.18a96c3b74e87384 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: 2f05b77a-6cf1-4cfa-9051-3fe201e43fd3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885 namespace: kserve-ci-e2e-test resourceVersion: "14803" uid: f9c13776-664e-492c-a0f5-c77316f6ec60 kind: Event lastTimestamp: "2026-04-24T22:40:29Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885-67946' metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:29Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885.18a96c23e75f5d83 namespace: kserve-ci-e2e-test resourceVersion: "14833" uid: f7d1f671-1f33-4c86-979d-a70aeb80fa9e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-133b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14799" uid: 82757d91-e0ae-4b73-a700-28baca8999dc kind: Event lastTimestamp: "2026-04-24T22:40:29Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-133b5-predictor-7cf69f4885 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:29Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor.18a96c23e0911b38 namespace: kserve-ci-e2e-test resourceVersion: "14806" uid: 5a8dd5af-cd95-45a8-9bf9-79657b602054 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:40:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-133b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14801" uid: a47a3f2c-8159-4f2c-b6c8-5cdbe63765d5 kind: Event lastTimestamp: "2026-04-24T22:41:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:14Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor.18a96c275f179fa6 namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: 5bea9f88-be1d-46ba-8ae6-5f56979743b4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:40:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-133b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14801" uid: a47a3f2c-8159-4f2c-b6c8-5cdbe63765d5 kind: Event lastTimestamp: "2026-04-24T22:41:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:14Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor.18a96c275f7d64ac namespace: kserve-ci-e2e-test resourceVersion: "15250" uid: b918b1a1-cfbf-47b7-a65f-5dcd015259a6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T22:40:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-133b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15021" uid: a47a3f2c-8159-4f2c-b6c8-5cdbe63765d5 kind: Event lastTimestamp: "2026-04-24T22:41:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:59Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor.18a96c2ade715747 namespace: kserve-ci-e2e-test resourceVersion: "15618" uid: 763598e8-4517-4c39-aabd-59f247887049 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T22:40:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-133b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15021" uid: a47a3f2c-8159-4f2c-b6c8-5cdbe63765d5 kind: Event lastTimestamp: "2026-04-24T22:41:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:59Z" name: isvc-xgboost-graph-raw-hpa-133b5-predictor.18a96c2adf1396c2 namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: 0ba4b088-3eca-43c6-8ad1-0b12610e1ad2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-133b5 namespace: kserve-ci-e2e-test resourceVersion: "14781" uid: d8ddbcc6-08a4-498f-be98-e002e5053126 kind: Event lastTimestamp: "2026-04-24T22:41:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:41:31Z" name: isvc-xgboost-graph-raw-hpa-133b5.18a96c23e05399db namespace: kserve-ci-e2e-test resourceVersion: "15385" uid: 6daa2276-dd52-4ce7-a12b-a5faae830b34 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-133b5 namespace: kserve-ci-e2e-test resourceVersion: "14781" uid: d8ddbcc6-08a4-498f-be98-e002e5053126 kind: Event lastTimestamp: "2026-04-24T22:40:29Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-133b5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-133b5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:40:29Z" name: isvc-xgboost-graph-raw-hpa-133b5.18a96c23e0ccf95a namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: 8c4734a7-82e4-42c8-959a-5de44baf26bf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-133b5 namespace: kserve-ci-e2e-test resourceVersion: "15383" uid: d8ddbcc6-08a4-498f-be98-e002e5053126 kind: Event lastTimestamp: "2026-04-24T22:41:31Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-133b5] is Ready metadata: creationTimestamp: "2026-04-24T22:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:41:31Z" name: isvc-xgboost-graph-raw-hpa-133b5.18a96c32660a1749 namespace: kserve-ci-e2e-test resourceVersion: "15384" uid: 28703457-79eb-424d-8e9c-f1f2bbf1985e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:42:09.553566Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-d7349-predictor-77567676df-jcnx9 namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: 7cc35065-8b02-4a65-b7c4-04c94f9ff71b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-d7349-predictor-77567676df-jcnx9 to ip-10-0-137-103.ec2.internal metadata: creationTimestamp: "2026-04-24T22:42:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:42:09Z" name: message-dumper-raw-d7349-predictor-77567676df-jcnx9.18a96c3b36db1ee3 namespace: kserve-ci-e2e-test resourceVersion: "15782" uid: cc0457ef-6aad-4338-986f-86973a496352 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58c7c775c7-zxnqk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:09Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-d7349-predictor-77567676df-jcnx9 namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: 7cc35065-8b02-4a65-b7c4-04c94f9ff71b kind: Event lastTimestamp: "2026-04-24T22:42:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-raw-d7349-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T22:42:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:09Z" name: message-dumper-raw-d7349-predictor-77567676df-jcnx9.18a96c3b3e8928c6 namespace: kserve-ci-e2e-test resourceVersion: "15792" uid: a19390f6-0360-4bed-aaac-286e4fdf5272 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:10Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-d7349-predictor-77567676df-jcnx9 namespace: kserve-ci-e2e-test resourceVersion: "15787" uid: 7cc35065-8b02-4a65-b7c4-04c94f9ff71b kind: Event lastTimestamp: "2026-04-24T22:42:10Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:42:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:42:10Z" name: message-dumper-raw-d7349-predictor-77567676df-jcnx9.18a96c3b735a4b79 namespace: kserve-ci-e2e-test resourceVersion: "15804" uid: ea070760-b1f0-4eb7-86d3-7cff2ff831ed reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-d7349-predictor-77567676df-jcnx9 namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: 7cc35065-8b02-4a65-b7c4-04c94f9ff71b kind: Event lastTimestamp: "2026-04-24T22:42:10Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-24T22:42:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:10Z" name: message-dumper-raw-d7349-predictor-77567676df-jcnx9.18a96c3b749d9754 namespace: kserve-ci-e2e-test resourceVersion: "15807" uid: 25568922-7bc9-4d55-8e43-e6baa0c7260d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-d7349-predictor-77567676df-jcnx9 namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: 7cc35065-8b02-4a65-b7c4-04c94f9ff71b kind: Event lastTimestamp: "2026-04-24T22:42:11Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.054s (1.054s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-24T22:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:11Z" name: message-dumper-raw-d7349-predictor-77567676df-jcnx9.18a96c3bb37f09cd namespace: kserve-ci-e2e-test resourceVersion: "15817" uid: 9ab0affa-e32c-4742-9545-dc2020069981 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-d7349-predictor-77567676df-jcnx9 namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: 7cc35065-8b02-4a65-b7c4-04c94f9ff71b kind: Event lastTimestamp: "2026-04-24T22:42:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:11Z" name: message-dumper-raw-d7349-predictor-77567676df-jcnx9.18a96c3bb6813ab4 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: 825c0ffe-c696-4c7c-8d2e-9099aea83dcc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-d7349-predictor-77567676df-jcnx9 namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: 7cc35065-8b02-4a65-b7c4-04c94f9ff71b kind: Event lastTimestamp: "2026-04-24T22:42:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:11Z" name: message-dumper-raw-d7349-predictor-77567676df-jcnx9.18a96c3bb6efcea8 namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: 09fb77a2-859f-4ceb-a051-497f251d970b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-d7349-predictor-77567676df-jcnx9 namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: 7cc35065-8b02-4a65-b7c4-04c94f9ff71b kind: Event lastTimestamp: "2026-04-24T22:42:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:11Z" name: message-dumper-raw-d7349-predictor-77567676df-jcnx9.18a96c3bb6fae3a5 namespace: kserve-ci-e2e-test resourceVersion: "15821" uid: b4cf4788-f1e4-40ea-91e7-d05a4a26f652 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-d7349-predictor-77567676df-jcnx9 namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: 7cc35065-8b02-4a65-b7c4-04c94f9ff71b kind: Event lastTimestamp: "2026-04-24T22:42:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:11Z" name: message-dumper-raw-d7349-predictor-77567676df-jcnx9.18a96c3bb9c3f0a1 namespace: kserve-ci-e2e-test resourceVersion: "15823" uid: e0325fc0-19bd-4740-91cc-92298a14e6d9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-d7349-predictor-77567676df-jcnx9 namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: 7cc35065-8b02-4a65-b7c4-04c94f9ff71b kind: Event lastTimestamp: "2026-04-24T22:42:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:11Z" name: message-dumper-raw-d7349-predictor-77567676df-jcnx9.18a96c3bba2d2ca4 namespace: kserve-ci-e2e-test resourceVersion: "15824" uid: 983e4324-eb92-48b0-80be-00526cdb33e9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:43:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-d7349-predictor-77567676df-jcnx9 namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: 7cc35065-8b02-4a65-b7c4-04c94f9ff71b kind: Event lastTimestamp: "2026-04-24T22:43:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:43:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:43:45Z" name: message-dumper-raw-d7349-predictor-77567676df-jcnx9.18a96c5170965aaf namespace: kserve-ci-e2e-test resourceVersion: "16664" uid: 32b9dbc5-1a5e-4009-bb33-d7b9c09b38c9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:43:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-d7349-predictor-77567676df-jcnx9 namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: 7cc35065-8b02-4a65-b7c4-04c94f9ff71b kind: Event lastTimestamp: "2026-04-24T22:43:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:43:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:43:45Z" name: message-dumper-raw-d7349-predictor-77567676df-jcnx9.18a96c517096d995 namespace: kserve-ci-e2e-test resourceVersion: "16665" uid: a5f9404f-c53c-496a-b1b0-aa0de45e3a62 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-d7349-predictor-77567676df namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: cb683b18-280d-403f-ad06-235329233bab kind: Event lastTimestamp: "2026-04-24T22:42:09Z" message: 'Created pod: message-dumper-raw-d7349-predictor-77567676df-jcnx9' metadata: creationTimestamp: "2026-04-24T22:42:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:42:09Z" name: message-dumper-raw-d7349-predictor-77567676df.18a96c3b365333e3 namespace: kserve-ci-e2e-test resourceVersion: "15776" uid: 2a95263f-55f3-44d6-a614-a9ad25438c87 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-d7349-predictor namespace: kserve-ci-e2e-test resourceVersion: "15747" uid: 7660eb29-b971-42a2-a161-05112716207d kind: Event lastTimestamp: "2026-04-24T22:42:09Z" message: Scaled up replica set message-dumper-raw-d7349-predictor-77567676df from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:42:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:42:09Z" name: message-dumper-raw-d7349-predictor.18a96c3b3323227f namespace: kserve-ci-e2e-test resourceVersion: "15757" uid: 45cd8108-6ebd-427f-af30-97a648371a91 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:42:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-d7349-predictor namespace: kserve-ci-e2e-test resourceVersion: "15752" uid: eb67119a-d594-49cc-9131-69526c24c18a kind: Event lastTimestamp: "2026-04-24T22:42:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:42:39Z" name: message-dumper-raw-d7349-predictor.18a96c3eb1e5b9ef namespace: kserve-ci-e2e-test resourceVersion: "16122" uid: 31e26324-1579-4bd4-9c8a-8cd312ddda4f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:42:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-d7349-predictor namespace: kserve-ci-e2e-test resourceVersion: "15752" uid: eb67119a-d594-49cc-9131-69526c24c18a kind: Event lastTimestamp: "2026-04-24T22:42:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:42:39Z" name: message-dumper-raw-d7349-predictor.18a96c3eb25338f2 namespace: kserve-ci-e2e-test resourceVersion: "16123" uid: 41350ce1-835a-4b74-89c9-26ba43b36423 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T22:42:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-d7349 namespace: kserve-ci-e2e-test resourceVersion: "15723" uid: aa163c74-2190-44e0-820b-4612ac5b210b kind: Event lastTimestamp: "2026-04-24T22:42:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:42:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:42:19Z" name: message-dumper-raw-d7349.18a96c3b33250951 namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: ff6ae5a8-8817-4c44-b6be-21fc2d56e8b6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-d7349 namespace: kserve-ci-e2e-test resourceVersion: "15756" uid: aa163c74-2190-44e0-820b-4612ac5b210b kind: Event lastTimestamp: "2026-04-24T22:42:09Z" message: 'Failed to update status for InferenceService "message-dumper-raw-d7349": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-d7349": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:42:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:42:09Z" name: message-dumper-raw-d7349.18a96c3b36734f1d namespace: kserve-ci-e2e-test resourceVersion: "15779" uid: 85c865aa-6069-4884-8842-89f99d41bdd3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-d7349 namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: aa163c74-2190-44e0-820b-4612ac5b210b kind: Event lastTimestamp: "2026-04-24T22:42:19Z" message: InferenceService [message-dumper-raw-d7349] is Ready metadata: creationTimestamp: "2026-04-24T22:42:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:42:19Z" name: message-dumper-raw-d7349.18a96c3d6a1ec344 namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: 8f0ea11a-b3a1-4877-8a6a-a3a2b7ee9feb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:40:19.046479Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-d4694-69669455cb-t9wzg namespace: kserve-ci-e2e-test resourceVersion: "14616" uid: 3cb48c49-1c88-4662-a812-ceca96ed4fc7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-d4694-69669455cb-t9wzg to ip-10-0-135-222.ec2.internal metadata: creationTimestamp: "2026-04-24T22:40:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:40:19Z" name: model-chainer-raw-d4694-69669455cb-t9wzg.18a96c217c1ee345 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 711c843e-25f0-4806-9a42-514716318fa1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58c7c775c7-zxnqk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:19Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-d4694-69669455cb-t9wzg namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 3cb48c49-1c88-4662-a812-ceca96ed4fc7 kind: Event lastTimestamp: "2026-04-24T22:40:19Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:40:19Z" name: model-chainer-raw-d4694-69669455cb-t9wzg.18a96c2195223bba namespace: kserve-ci-e2e-test resourceVersion: "14642" uid: 2c9018ce-7d7b-4221-aa6c-c1e8fc9b4049 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-d4694} kind: Pod name: model-chainer-raw-d4694-69669455cb-t9wzg namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: 3cb48c49-1c88-4662-a812-ceca96ed4fc7 kind: Event lastTimestamp: "2026-04-24T22:40:19Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:f3ef58ae57fd8b984890d9aeb7935958f0ea100fbddeb0833d76b7e2007120ff" metadata: creationTimestamp: "2026-04-24T22:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:19Z" name: model-chainer-raw-d4694-69669455cb-t9wzg.18a96c219676814e namespace: kserve-ci-e2e-test resourceVersion: "14644" uid: 9703be97-2c32-41c0-8c78-81c5c9175f63 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-222.ec2.internal source: component: kubelet host: ip-10-0-135-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-d4694} kind: Pod name: model-chainer-raw-d4694-69669455cb-t9wzg namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: 3cb48c49-1c88-4662-a812-ceca96ed4fc7 kind: Event lastTimestamp: "2026-04-24T22:40:21Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:f3ef58ae57fd8b984890d9aeb7935958f0ea100fbddeb0833d76b7e2007120ff" in 1.649s (1.649s including waiting). Image size: 216346192 bytes.' metadata: creationTimestamp: "2026-04-24T22:40:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:21Z" name: model-chainer-raw-d4694-69669455cb-t9wzg.18a96c21f8cbb1f3 namespace: kserve-ci-e2e-test resourceVersion: "14658" uid: a45076ba-110d-49af-8243-f96731500b8b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-222.ec2.internal source: component: kubelet host: ip-10-0-135-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-d4694} kind: Pod name: model-chainer-raw-d4694-69669455cb-t9wzg namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: 3cb48c49-1c88-4662-a812-ceca96ed4fc7 kind: Event lastTimestamp: "2026-04-24T22:40:21Z" message: 'Created container: model-chainer-raw-d4694' metadata: creationTimestamp: "2026-04-24T22:40:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:21Z" name: model-chainer-raw-d4694-69669455cb-t9wzg.18a96c21fc0bdc7d namespace: kserve-ci-e2e-test resourceVersion: "14659" uid: d35384c1-d1d7-430c-a411-eb94fcb3b24b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-222.ec2.internal source: component: kubelet host: ip-10-0-135-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-d4694} kind: Pod name: model-chainer-raw-d4694-69669455cb-t9wzg namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: 3cb48c49-1c88-4662-a812-ceca96ed4fc7 kind: Event lastTimestamp: "2026-04-24T22:40:21Z" message: Started container model-chainer-raw-d4694 metadata: creationTimestamp: "2026-04-24T22:40:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:21Z" name: model-chainer-raw-d4694-69669455cb-t9wzg.18a96c21fc945171 namespace: kserve-ci-e2e-test resourceVersion: "14660" uid: a93b69cc-83bf-4e46-b4de-887e35802043 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-222.ec2.internal source: component: kubelet host: ip-10-0-135-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-d4694} kind: Pod name: model-chainer-raw-d4694-69669455cb-t9wzg namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: 3cb48c49-1c88-4662-a812-ceca96ed4fc7 kind: Event lastTimestamp: "2026-04-24T22:40:29Z" message: Stopping container model-chainer-raw-d4694 metadata: creationTimestamp: "2026-04-24T22:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:29Z" name: model-chainer-raw-d4694-69669455cb-t9wzg.18a96c23d1c63fad namespace: kserve-ci-e2e-test resourceVersion: "14738" uid: e7a7df91-f450-49b2-84c1-076d19994a55 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-222.ec2.internal source: component: kubelet host: ip-10-0-135-222.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:40:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-d4694} kind: Pod name: model-chainer-raw-d4694-69669455cb-t9wzg namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: 3cb48c49-1c88-4662-a812-ceca96ed4fc7 kind: Event lastTimestamp: "2026-04-24T22:40:57Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T22:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:57Z" name: model-chainer-raw-d4694-69669455cb-t9wzg.18a96c24ac166ba8 namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: db812d30-d840-4006-af7b-04fe2255f1fc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-222.ec2.internal source: component: kubelet host: ip-10-0-135-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-d4694-69669455cb namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: 2db9e15a-badb-4cc4-b8d6-6545e150bac2 kind: Event lastTimestamp: "2026-04-24T22:40:19Z" message: 'Created pod: model-chainer-raw-d4694-69669455cb-t9wzg' metadata: creationTimestamp: "2026-04-24T22:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:19Z" name: model-chainer-raw-d4694-69669455cb.18a96c217ba53683 namespace: kserve-ci-e2e-test resourceVersion: "14619" uid: 542c7e7f-4afb-4630-8fe5-15d09929e118 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-d4694 namespace: kserve-ci-e2e-test resourceVersion: "14612" uid: 30be919a-aed5-4d10-81dd-1056a7a3465f kind: Event lastTimestamp: "2026-04-24T22:40:19Z" message: Scaled up replica set model-chainer-raw-d4694-69669455cb from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:19Z" name: model-chainer-raw-d4694.18a96c217a88fe45 namespace: kserve-ci-e2e-test resourceVersion: "14617" uid: 26817e82-32e2-4bfd-b2e4-fb3329af30e0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-d4694 namespace: kserve-ci-e2e-test resourceVersion: "14633" uid: 2e83d178-20a5-4bc1-b894-de2331dde992 kind: Event lastTimestamp: "2026-04-24T22:40:19Z" message: InferenceGraph [model-chainer-raw-d4694] is Ready metadata: creationTimestamp: "2026-04-24T22:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:40:19Z" name: model-chainer-raw-d4694.18a96c217ff0934c namespace: kserve-ci-e2e-test resourceVersion: "14635" uid: 5034be02-64de-4fe0-b95c-019894b94959 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:41:59.317253Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-133b5-7bdbb47f56-9svl5 namespace: kserve-ci-e2e-test resourceVersion: "15600" uid: 30fc9e5b-d308-4796-8b2c-a69ec1e808ab kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-133b5-7bdbb47f56-9svl5 to ip-10-0-135-222.ec2.internal metadata: creationTimestamp: "2026-04-24T22:41:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:41:59Z" name: model-chainer-raw-hpa-133b5-7bdbb47f56-9svl5.18a96c38d4b96695 namespace: kserve-ci-e2e-test resourceVersion: "15605" uid: 291cfeae-a5e0-46ca-9d4e-fa2eb84c85bb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58c7c775c7-zxnqk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:59Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-133b5-7bdbb47f56-9svl5 namespace: kserve-ci-e2e-test resourceVersion: "15610" uid: 30fc9e5b-d308-4796-8b2c-a69ec1e808ab kind: Event lastTimestamp: "2026-04-24T22:41:59Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:41:59Z" name: model-chainer-raw-hpa-133b5-7bdbb47f56-9svl5.18a96c38ee0811c3 namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: d192a074-95bb-46ce-8ab8-0cdc1e43b867 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-133b5} kind: Pod name: model-chainer-raw-hpa-133b5-7bdbb47f56-9svl5 namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: 30fc9e5b-d308-4796-8b2c-a69ec1e808ab kind: Event lastTimestamp: "2026-04-24T22:41:59Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f3ef58ae57fd8b984890d9aeb7935958f0ea100fbddeb0833d76b7e2007120ff" already present on machine metadata: creationTimestamp: "2026-04-24T22:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:41:59Z" name: model-chainer-raw-hpa-133b5-7bdbb47f56-9svl5.18a96c38ef4c043f namespace: kserve-ci-e2e-test resourceVersion: "15628" uid: 2a71b959-2752-495d-a34b-f3a27104384b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-222.ec2.internal source: component: kubelet host: ip-10-0-135-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-133b5} kind: Pod name: model-chainer-raw-hpa-133b5-7bdbb47f56-9svl5 namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: 30fc9e5b-d308-4796-8b2c-a69ec1e808ab kind: Event lastTimestamp: "2026-04-24T22:41:59Z" message: 'Created container: model-chainer-raw-hpa-133b5' metadata: creationTimestamp: "2026-04-24T22:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:41:59Z" name: model-chainer-raw-hpa-133b5-7bdbb47f56-9svl5.18a96c38f1fef014 namespace: kserve-ci-e2e-test resourceVersion: "15629" uid: 42e1b37e-97c0-4bd7-a4f9-51df6432c3ac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-222.ec2.internal source: component: kubelet host: ip-10-0-135-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-133b5} kind: Pod name: model-chainer-raw-hpa-133b5-7bdbb47f56-9svl5 namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: 30fc9e5b-d308-4796-8b2c-a69ec1e808ab kind: Event lastTimestamp: "2026-04-24T22:41:59Z" message: Started container model-chainer-raw-hpa-133b5 metadata: creationTimestamp: "2026-04-24T22:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:41:59Z" name: model-chainer-raw-hpa-133b5-7bdbb47f56-9svl5.18a96c38f2881ca6 namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: a5df4c34-5387-495e-a991-56909022d731 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-222.ec2.internal source: component: kubelet host: ip-10-0-135-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:42:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-133b5} kind: Pod name: model-chainer-raw-hpa-133b5-7bdbb47f56-9svl5 namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: 30fc9e5b-d308-4796-8b2c-a69ec1e808ab kind: Event lastTimestamp: "2026-04-24T22:42:09Z" message: Stopping container model-chainer-raw-hpa-133b5 metadata: creationTimestamp: "2026-04-24T22:42:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:09Z" name: model-chainer-raw-hpa-133b5-7bdbb47f56-9svl5.18a96c3b2baee8ae namespace: kserve-ci-e2e-test resourceVersion: "15720" uid: e2026b3a-4ede-455a-9001-9405de10d507 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-222.ec2.internal source: component: kubelet host: ip-10-0-135-222.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-133b5} kind: Pod name: model-chainer-raw-hpa-133b5-7bdbb47f56-9svl5 namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: 30fc9e5b-d308-4796-8b2c-a69ec1e808ab kind: Event lastTimestamp: "2026-04-24T22:42:36Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T22:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:42:36Z" name: model-chainer-raw-hpa-133b5-7bdbb47f56-9svl5.18a96c3b8fff26ed namespace: kserve-ci-e2e-test resourceVersion: "16094" uid: 685b335c-c8d0-4c89-9653-93faeee16cbc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-222.ec2.internal source: component: kubelet host: ip-10-0-135-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-133b5-7bdbb47f56 namespace: kserve-ci-e2e-test resourceVersion: "15597" uid: 92dc2277-d5e6-4a1f-86ba-97bbfc3aa65e kind: Event lastTimestamp: "2026-04-24T22:41:59Z" message: 'Created pod: model-chainer-raw-hpa-133b5-7bdbb47f56-9svl5' metadata: creationTimestamp: "2026-04-24T22:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:59Z" name: model-chainer-raw-hpa-133b5-7bdbb47f56.18a96c38d4307ccc namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 81596879-09ac-4ba5-a2e3-f2608815220c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-133b5 namespace: kserve-ci-e2e-test resourceVersion: "15595" uid: 0a22adb1-d9c3-435a-b791-d8c06554f4a5 kind: Event lastTimestamp: "2026-04-24T22:41:59Z" message: Scaled up replica set model-chainer-raw-hpa-133b5-7bdbb47f56 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:59Z" name: model-chainer-raw-hpa-133b5.18a96c38d355e3f9 namespace: kserve-ci-e2e-test resourceVersion: "15598" uid: bc81ffab-5854-4b2d-be9b-9d24b4553460 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:59Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-133b5 namespace: kserve-ci-e2e-test resourceVersion: "15613" uid: 034d0b63-1a42-4e6d-ab79-dc1ff37d666c kind: Event lastTimestamp: "2026-04-24T22:41:59Z" message: InferenceGraph [model-chainer-raw-hpa-133b5] is Ready metadata: creationTimestamp: "2026-04-24T22:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:41:59Z" name: model-chainer-raw-hpa-133b5.18a96c38d8d27fcf namespace: kserve-ci-e2e-test resourceVersion: "15615" uid: 82a5539a-4aa2-49f2-9d04-4164c671570d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:59Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-133b5 namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 034d0b63-1a42-4e6d-ab79-dc1ff37d666c kind: Event lastTimestamp: "2026-04-24T22:41:59Z" message: 'Failed to update status for InferenceGraph "model-chainer-raw-hpa-133b5": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-hpa-133b5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:41:59Z" name: model-chainer-raw-hpa-133b5.18a96c38db3f8547 namespace: kserve-ci-e2e-test resourceVersion: "15620" uid: 898f76f5-2559-4971-8ebc-5edf43885eb6 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:59Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-133b5 namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 034d0b63-1a42-4e6d-ab79-dc1ff37d666c kind: Event lastTimestamp: "2026-04-24T22:41:59Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-hpa-133b5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:41:59Z" name: model-chainer-raw-hpa-133b5.18a96c38db3febd6 namespace: kserve-ci-e2e-test resourceVersion: "15621" uid: 2b06042e-9ec2-49ad-9087-fcf86e5bc0aa reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:47:34.584099Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l namespace: kserve-ci-e2e-test resourceVersion: "18649" uid: 595d3465-735f-451b-8411-9913eb253a29 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l to ip-10-0-137-103.ec2.internal metadata: creationTimestamp: "2026-04-24T22:47:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:47:34Z" name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l.18a96c86e42f8aec namespace: kserve-ci-e2e-test resourceVersion: "18653" uid: 65742a1b-a5ea-4a11-9a09-bcbc8a5cf0f8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58c7c775c7-zxnqk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:34Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 595d3465-735f-451b-8411-9913eb253a29 kind: Event lastTimestamp: "2026-04-24T22:47:34Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-b6db3-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T22:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:34Z" name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l.18a96c86f0b1af7d namespace: kserve-ci-e2e-test resourceVersion: "18663" uid: cce7f7d4-9c17-4fd3-a1fb-144a13031a56 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:35Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l namespace: kserve-ci-e2e-test resourceVersion: "18659" uid: 595d3465-735f-451b-8411-9913eb253a29 kind: Event lastTimestamp: "2026-04-24T22:47:35Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:47:35Z" name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l.18a96c872094ea18 namespace: kserve-ci-e2e-test resourceVersion: "18678" uid: 5ba2c90b-b065-4a59-8269-8b888945e2d3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 595d3465-735f-451b-8411-9913eb253a29 kind: Event lastTimestamp: "2026-04-24T22:47:35Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:4ae4df75945dd799b0c11b5e37a48a5ba8230ff3507d345ea07c8897a8c39885" already present on machine metadata: creationTimestamp: "2026-04-24T22:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:35Z" name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l.18a96c8721e0c330 namespace: kserve-ci-e2e-test resourceVersion: "18680" uid: 94b87341-99a0-491f-aba7-5cff0979eda5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 595d3465-735f-451b-8411-9913eb253a29 kind: Event lastTimestamp: "2026-04-24T22:47:35Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T22:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:35Z" name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l.18a96c8724d37325 namespace: kserve-ci-e2e-test resourceVersion: "18681" uid: b0074ee3-dbb6-4ee8-beea-d2053bff746b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 595d3465-735f-451b-8411-9913eb253a29 kind: Event lastTimestamp: "2026-04-24T22:47:35Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T22:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:35Z" name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l.18a96c87254af62a namespace: kserve-ci-e2e-test resourceVersion: "18683" uid: b7aee4e3-f485-4586-8a9d-56934e8e364c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 595d3465-735f-451b-8411-9913eb253a29 kind: Event lastTimestamp: "2026-04-24T22:47:39Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:47:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:39Z" name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l.18a96c8824fc7a8b namespace: kserve-ci-e2e-test resourceVersion: "18719" uid: b3408b5f-b22a-4e89-87c3-b083debbfc82 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 595d3465-735f-451b-8411-9913eb253a29 kind: Event lastTimestamp: "2026-04-24T22:47:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:40Z" name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l.18a96c8827f2cf80 namespace: kserve-ci-e2e-test resourceVersion: "18721" uid: 1668b4ba-c4b1-4053-a35b-d850acf5a923 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 595d3465-735f-451b-8411-9913eb253a29 kind: Event lastTimestamp: "2026-04-24T22:47:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:40Z" name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l.18a96c882861b14c namespace: kserve-ci-e2e-test resourceVersion: "18722" uid: 7e1e6355-665f-4a80-b729-314d13e9c6db reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 595d3465-735f-451b-8411-9913eb253a29 kind: Event lastTimestamp: "2026-04-24T22:47:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:40Z" name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l.18a96c88286c50e7 namespace: kserve-ci-e2e-test resourceVersion: "18724" uid: 5e8d950d-e8b4-451e-be53-7b7f6d8fde2d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 595d3465-735f-451b-8411-9913eb253a29 kind: Event lastTimestamp: "2026-04-24T22:47:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:40Z" name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l.18a96c882b162650 namespace: kserve-ci-e2e-test resourceVersion: "18726" uid: 0798124d-ab4a-46e9-a281-17c3195c0927 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 595d3465-735f-451b-8411-9913eb253a29 kind: Event lastTimestamp: "2026-04-24T22:47:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:47:40Z" name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l.18a96c882b7fede4 namespace: kserve-ci-e2e-test resourceVersion: "18728" uid: 9a368a75-f228-41e0-aa54-9971bf5573df reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T22:47:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 595d3465-735f-451b-8411-9913eb253a29 kind: Event lastTimestamp: "2026-04-24T22:48:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:48:57Z" name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l.18a96c889cbf8a4d namespace: kserve-ci-e2e-test resourceVersion: "19354" uid: d5fbeff0-a6a7-4f08-8deb-01984b832066 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:48:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 595d3465-735f-451b-8411-9913eb253a29 kind: Event lastTimestamp: "2026-04-24T22:48:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:48:54Z" name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l.18a96c9988482621 namespace: kserve-ci-e2e-test resourceVersion: "19285" uid: 2aedfb2e-b92e-4e97-a811-8135068cc177 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:48:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 595d3465-735f-451b-8411-9913eb253a29 kind: Event lastTimestamp: "2026-04-24T22:48:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:48:54Z" name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l.18a96c99884887a2 namespace: kserve-ci-e2e-test resourceVersion: "19288" uid: 42ee99f2-0c7f-4357-b417-b5e730821f43 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:48:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 595d3465-735f-451b-8411-9913eb253a29 kind: Event lastTimestamp: "2026-04-24T22:48:57Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:48:57Z" name: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l.18a96c9a4eda7c8f namespace: kserve-ci-e2e-test resourceVersion: "19353" uid: a9f611e3-7f6e-4369-abaa-1bad98d59d02 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-b6db3-predictor-6544bf9b66 namespace: kserve-ci-e2e-test resourceVersion: "18638" uid: 39b2eb1b-3cec-4cc1-9e5c-83cd128700d9 kind: Event lastTimestamp: "2026-04-24T22:47:34Z" message: 'Created pod: raw-sklearn-b6db3-predictor-6544bf9b66-j6v7l' metadata: creationTimestamp: "2026-04-24T22:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:47:34Z" name: raw-sklearn-b6db3-predictor-6544bf9b66.18a96c86e39ee887 namespace: kserve-ci-e2e-test resourceVersion: "18650" uid: 76384371-05de-4219-819f-378c14c49af8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-b6db3-predictor namespace: kserve-ci-e2e-test resourceVersion: "18636" uid: 31f89176-7732-4a6f-a0a6-2eda461c364f kind: Event lastTimestamp: "2026-04-24T22:47:34Z" message: Scaled up replica set raw-sklearn-b6db3-predictor-6544bf9b66 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:47:34Z" name: raw-sklearn-b6db3-predictor.18a96c86e0d33762 namespace: kserve-ci-e2e-test resourceVersion: "18641" uid: cf500713-2af4-456e-9e29-c7cdd6cefc90 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:47:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-b6db3-predictor namespace: kserve-ci-e2e-test resourceVersion: "18637" uid: 515ff6b8-8899-44f8-9d1b-7c4a8d66d3b6 kind: Event lastTimestamp: "2026-04-24T22:48:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:48:19Z" name: raw-sklearn-b6db3-predictor.18a96c8a5fdfeca6 namespace: kserve-ci-e2e-test resourceVersion: "19006" uid: 1d1b396c-8e5b-41d9-b1b4-35fda87308e9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:47:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-b6db3-predictor namespace: kserve-ci-e2e-test resourceVersion: "18637" uid: 515ff6b8-8899-44f8-9d1b-7c4a8d66d3b6 kind: Event lastTimestamp: "2026-04-24T22:48:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:48:19Z" name: raw-sklearn-b6db3-predictor.18a96c8a604f9d78 namespace: kserve-ci-e2e-test resourceVersion: "19007" uid: 77902daa-6bd3-46fc-90a9-800dc83f059b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:48:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-b6db3-predictor namespace: kserve-ci-e2e-test resourceVersion: "18798" uid: 515ff6b8-8899-44f8-9d1b-7c4a8d66d3b6 kind: Event lastTimestamp: "2026-04-24T22:48:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:48:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:48:49Z" name: raw-sklearn-b6db3-predictor.18a96c94df08d78f namespace: kserve-ci-e2e-test resourceVersion: "19224" uid: 3e3e5317-ff70-4b15-b435-d8b7948f181d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:48:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-b6db3-predictor namespace: kserve-ci-e2e-test resourceVersion: "18798" uid: 515ff6b8-8899-44f8-9d1b-7c4a8d66d3b6 kind: Event lastTimestamp: "2026-04-24T22:48:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:48:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:48:49Z" name: raw-sklearn-b6db3-predictor.18a96c94df773022 namespace: kserve-ci-e2e-test resourceVersion: "19225" uid: b6ebae86-5d3b-41fc-9665-1156e4e6b093 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T22:47:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-b6db3 namespace: kserve-ci-e2e-test resourceVersion: "18625" uid: b02027d5-5486-46b9-b17e-410e4ab3ef5c kind: Event lastTimestamp: "2026-04-24T22:48:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:48:48Z" name: raw-sklearn-b6db3.18a96c86e0a9aa64 namespace: kserve-ci-e2e-test resourceVersion: "19214" uid: f12c67dd-e129-423a-aa01-af39036cf316 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:47:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-b6db3 namespace: kserve-ci-e2e-test resourceVersion: "18625" uid: b02027d5-5486-46b9-b17e-410e4ab3ef5c kind: Event lastTimestamp: "2026-04-24T22:47:34Z" message: 'Failed to update status for InferenceService "raw-sklearn-b6db3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-b6db3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:47:34Z" name: raw-sklearn-b6db3.18a96c86e11f6409 namespace: kserve-ci-e2e-test resourceVersion: "18662" uid: 760ddd6f-397a-4c1f-9911-65e6c1e5e57c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:47:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-b6db3 namespace: kserve-ci-e2e-test resourceVersion: "18647" uid: b02027d5-5486-46b9-b17e-410e4ab3ef5c kind: Event lastTimestamp: "2026-04-24T22:47:34Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-b6db3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:47:34Z" name: raw-sklearn-b6db3.18a96c86e863d56c namespace: kserve-ci-e2e-test resourceVersion: "18664" uid: 0ae94527-ffca-46e8-8936-54f9e02f93be reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:48:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-b6db3 namespace: kserve-ci-e2e-test resourceVersion: "19210" uid: b02027d5-5486-46b9-b17e-410e4ab3ef5c kind: Event lastTimestamp: "2026-04-24T22:48:48Z" message: InferenceService [raw-sklearn-b6db3] is Ready metadata: creationTimestamp: "2026-04-24T22:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:48:48Z" name: raw-sklearn-b6db3.18a96c980001321e namespace: kserve-ci-e2e-test resourceVersion: "19212" uid: 71bb5ac0-5ebe-4695-b9d1-2bebd9485f31 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:48:54.733156Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z namespace: kserve-ci-e2e-test resourceVersion: "19304" uid: 5af0f174-66f8-4ba9-a0fb-b05fed19fccf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z to ip-10-0-137-103.ec2.internal metadata: creationTimestamp: "2026-04-24T22:48:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:48:54Z" name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z.18a96c998d713ff6 namespace: kserve-ci-e2e-test resourceVersion: "19308" uid: 0feb2d0a-7c8d-4146-8aac-396ee14fcce6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58c7c775c7-zxnqk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:48:54Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 5af0f174-66f8-4ba9-a0fb-b05fed19fccf kind: Event lastTimestamp: "2026-04-24T22:48:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-runtime-4d001-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T22:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:48:54Z" name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z.18a96c999b0511a4 namespace: kserve-ci-e2e-test resourceVersion: "19316" uid: 4ef3084f-0677-4bbc-9777-b79bf53b69b4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:48:55Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z namespace: kserve-ci-e2e-test resourceVersion: "19312" uid: 5af0f174-66f8-4ba9-a0fb-b05fed19fccf kind: Event lastTimestamp: "2026-04-24T22:48:55Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:48:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:48:55Z" name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z.18a96c99ca138c9c namespace: kserve-ci-e2e-test resourceVersion: "19326" uid: b2bfac22-6d7c-444e-a33d-7dda317ec652 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:48:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 5af0f174-66f8-4ba9-a0fb-b05fed19fccf kind: Event lastTimestamp: "2026-04-24T22:48:55Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:4ae4df75945dd799b0c11b5e37a48a5ba8230ff3507d345ea07c8897a8c39885" already present on machine metadata: creationTimestamp: "2026-04-24T22:48:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:48:55Z" name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z.18a96c99cb53eec7 namespace: kserve-ci-e2e-test resourceVersion: "19328" uid: f704224b-3944-45ce-9b45-7422b84d79f9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:48:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 5af0f174-66f8-4ba9-a0fb-b05fed19fccf kind: Event lastTimestamp: "2026-04-24T22:48:55Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T22:48:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:48:55Z" name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z.18a96c99ce6a2f71 namespace: kserve-ci-e2e-test resourceVersion: "19330" uid: 66923bb2-5b8f-406a-ad76-4c3a0dec570b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:48:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 5af0f174-66f8-4ba9-a0fb-b05fed19fccf kind: Event lastTimestamp: "2026-04-24T22:48:55Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T22:48:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:48:55Z" name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z.18a96c99ced53c42 namespace: kserve-ci-e2e-test resourceVersion: "19331" uid: 4b1fb36a-5950-4556-855f-c97be053ae14 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 5af0f174-66f8-4ba9-a0fb-b05fed19fccf kind: Event lastTimestamp: "2026-04-24T22:49:00Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:00Z" name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z.18a96c9ad40e3252 namespace: kserve-ci-e2e-test resourceVersion: "19373" uid: 0257e688-b7e3-4c97-826f-c0faadcc7ba4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 5af0f174-66f8-4ba9-a0fb-b05fed19fccf kind: Event lastTimestamp: "2026-04-24T22:49:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:00Z" name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z.18a96c9ad6c9bf85 namespace: kserve-ci-e2e-test resourceVersion: "19375" uid: 6046029f-e2d5-4f38-b602-1107eb2cced6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 5af0f174-66f8-4ba9-a0fb-b05fed19fccf kind: Event lastTimestamp: "2026-04-24T22:49:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:00Z" name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z.18a96c9ad73a588b namespace: kserve-ci-e2e-test resourceVersion: "19376" uid: 4178e23e-7988-47c2-8145-d29b678605c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 5af0f174-66f8-4ba9-a0fb-b05fed19fccf kind: Event lastTimestamp: "2026-04-24T22:49:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:00Z" name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z.18a96c9ad745d278 namespace: kserve-ci-e2e-test resourceVersion: "19377" uid: 625280ea-4f5f-46f8-9661-bb61528576b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 5af0f174-66f8-4ba9-a0fb-b05fed19fccf kind: Event lastTimestamp: "2026-04-24T22:49:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:00Z" name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z.18a96c9ada553cf1 namespace: kserve-ci-e2e-test resourceVersion: "19381" uid: be593d83-b0c0-4a09-b668-0bbe5d02a9a6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 5af0f174-66f8-4ba9-a0fb-b05fed19fccf kind: Event lastTimestamp: "2026-04-24T22:49:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:00Z" name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z.18a96c9adac0c610 namespace: kserve-ci-e2e-test resourceVersion: "19382" uid: 27a1fcc3-a5d5-472a-a689-3cbbe6d91db5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T22:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 5af0f174-66f8-4ba9-a0fb-b05fed19fccf kind: Event lastTimestamp: "2026-04-24T22:50:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:18Z" name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z.18a96c9b4bba51df namespace: kserve-ci-e2e-test resourceVersion: "19997" uid: 5aa6b600-1975-4414-bd41-a36da16e930c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 5af0f174-66f8-4ba9-a0fb-b05fed19fccf kind: Event lastTimestamp: "2026-04-24T22:50:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:14Z" name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z.18a96cac32790b3a namespace: kserve-ci-e2e-test resourceVersion: "19933" uid: 6ef1e749-8276-4068-a6cd-104e699f4f23 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 5af0f174-66f8-4ba9-a0fb-b05fed19fccf kind: Event lastTimestamp: "2026-04-24T22:50:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:14Z" name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z.18a96cac3279c09c namespace: kserve-ci-e2e-test resourceVersion: "19934" uid: 8bd211c6-0d28-43a5-b8a0-4dee3a90493e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 5af0f174-66f8-4ba9-a0fb-b05fed19fccf kind: Event lastTimestamp: "2026-04-24T22:50:18Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:18Z" name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z.18a96cacfdcf642a namespace: kserve-ci-e2e-test resourceVersion: "19996" uid: 9f7fff72-032d-43fd-af9e-7e4f82b69439 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-103.ec2.internal source: component: kubelet host: ip-10-0-137-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:48:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6 namespace: kserve-ci-e2e-test resourceVersion: "19293" uid: 96d82862-c642-4dfc-812b-c435cd7b79e0 kind: Event lastTimestamp: "2026-04-24T22:48:54Z" message: 'Created pod: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6-frt5z' metadata: creationTimestamp: "2026-04-24T22:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:48:54Z" name: raw-sklearn-runtime-4d001-predictor-79b9cb5cf6.18a96c998d14c221 namespace: kserve-ci-e2e-test resourceVersion: "19305" uid: a291d469-fb5d-4e9e-af56-49575719c4b6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:48:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-4d001-predictor namespace: kserve-ci-e2e-test resourceVersion: "19291" uid: 68134b83-2516-4e2b-9d7b-d4beeb4c93d0 kind: Event lastTimestamp: "2026-04-24T22:48:54Z" message: Scaled up replica set raw-sklearn-runtime-4d001-predictor-79b9cb5cf6 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:48:54Z" name: raw-sklearn-runtime-4d001-predictor.18a96c9989eb263a namespace: kserve-ci-e2e-test resourceVersion: "19295" uid: 72ddb742-9644-4330-b531-37a37b67d693 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:49:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-4d001-predictor namespace: kserve-ci-e2e-test resourceVersion: "19292" uid: 0a9c1191-2262-45c9-80fe-536177c74d66 kind: Event lastTimestamp: "2026-04-24T22:49:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:49:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:49:24Z" name: raw-sklearn-runtime-4d001-predictor.18a96c9d090838d8 namespace: kserve-ci-e2e-test resourceVersion: "19554" uid: e3d65a0d-fc80-4cf9-8618-393435a6cfa8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:49:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-4d001-predictor namespace: kserve-ci-e2e-test resourceVersion: "19292" uid: 0a9c1191-2262-45c9-80fe-536177c74d66 kind: Event lastTimestamp: "2026-04-24T22:49:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:49:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:49:24Z" name: raw-sklearn-runtime-4d001-predictor.18a96c9d09775aeb namespace: kserve-ci-e2e-test resourceVersion: "19555" uid: b3804e1e-8feb-47b9-9c80-00e47420b420 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:49:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-4d001-predictor namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: 0a9c1191-2262-45c9-80fe-536177c74d66 kind: Event lastTimestamp: "2026-04-24T22:50:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:49:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:09Z" name: raw-sklearn-runtime-4d001-predictor.18a96ca40799054d namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: b91d854d-8c20-4020-bebe-980904781d6d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:49:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-4d001-predictor namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: 0a9c1191-2262-45c9-80fe-536177c74d66 kind: Event lastTimestamp: "2026-04-24T22:50:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:49:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:09Z" name: raw-sklearn-runtime-4d001-predictor.18a96ca4081222ea namespace: kserve-ci-e2e-test resourceVersion: "19885" uid: bb5f1c5a-6504-4ebf-9410-c731004024fd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T22:48:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-4d001 namespace: kserve-ci-e2e-test resourceVersion: "19262" uid: 27b83ef2-a254-4139-8af6-b7e4404aa1af kind: Event lastTimestamp: "2026-04-24T22:50:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:50:08Z" name: raw-sklearn-runtime-4d001.18a96c9989ca11c7 namespace: kserve-ci-e2e-test resourceVersion: "19872" uid: 65a8e239-ed1e-4b69-b9c9-adc0bcea88c8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:48:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-4d001 namespace: kserve-ci-e2e-test resourceVersion: "19262" uid: 27b83ef2-a254-4139-8af6-b7e4404aa1af kind: Event lastTimestamp: "2026-04-24T22:48:54Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-4d001": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-4d001": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:48:54Z" name: raw-sklearn-runtime-4d001.18a96c998a2fe589 namespace: kserve-ci-e2e-test resourceVersion: "19297" uid: dc2f3464-8f90-43c0-90f8-e793ac397640 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-4d001 namespace: kserve-ci-e2e-test resourceVersion: "19870" uid: 27b83ef2-a254-4139-8af6-b7e4404aa1af kind: Event lastTimestamp: "2026-04-24T22:50:08Z" message: InferenceService [raw-sklearn-runtime-4d001] is Ready metadata: creationTimestamp: "2026-04-24T22:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:50:08Z" name: raw-sklearn-runtime-4d001.18a96caaaeb417ac namespace: kserve-ci-e2e-test resourceVersion: "19871" uid: 7e6e5b64-68d7-435a-a459-c2341987f14c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "20125"