--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:07:39.146479Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-7ebb90-predictor-6b8ccc595-6qcz5 namespace: kserve-ci-e2e-test resourceVersion: "18844" uid: ca4c3af6-5616-42b6-9b89-ee283f3cf89b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-7ebb90-predictor-6b8ccc595-6qcz5 to ip-10-0-130-1.ec2.internal metadata: creationTimestamp: "2026-04-16T17:07:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:07:39Z" name: isvc-init-fail-7ebb90-predictor-6b8ccc595-6qcz5.18a6e555aae6fb41 namespace: kserve-ci-e2e-test resourceVersion: "18849" uid: 9143cc10-ccec-49a7-9280-1fae2ef01a91 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f7c99d688-jwz56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-7ebb90-predictor-6b8ccc595-6qcz5 namespace: kserve-ci-e2e-test resourceVersion: "18854" uid: ca4c3af6-5616-42b6-9b89-ee283f3cf89b kind: Event lastTimestamp: "2026-04-16T17:07:39Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:07:39Z" name: isvc-init-fail-7ebb90-predictor-6b8ccc595-6qcz5.18a6e555c4bf75eb namespace: kserve-ci-e2e-test resourceVersion: "18871" uid: 26f21c3d-1447-4361-ade5-bbcdd2bebf0d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:07:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-7ebb90-predictor-6b8ccc595-6qcz5 namespace: kserve-ci-e2e-test resourceVersion: "18846" uid: ca4c3af6-5616-42b6-9b89-ee283f3cf89b kind: Event lastTimestamp: "2026-04-16T17:07:43Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ab565fe3972fd5cd2929b41b0a6a9face626513b818fecb7a38edcdfe0857061" already present on machine metadata: creationTimestamp: "2026-04-16T17:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:07:43Z" name: isvc-init-fail-7ebb90-predictor-6b8ccc595-6qcz5.18a6e555c61261aa namespace: kserve-ci-e2e-test resourceVersion: "18913" uid: 7ea3431f-a68d-4174-896c-c4e61a80dd27 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:07:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-7ebb90-predictor-6b8ccc595-6qcz5 namespace: kserve-ci-e2e-test resourceVersion: "18846" uid: ca4c3af6-5616-42b6-9b89-ee283f3cf89b kind: Event lastTimestamp: "2026-04-16T17:07:43Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T17:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:07:43Z" name: isvc-init-fail-7ebb90-predictor-6b8ccc595-6qcz5.18a6e555c927121b namespace: kserve-ci-e2e-test resourceVersion: "18917" uid: 7e65590c-4d9a-4d16-a146-2540b1b262ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:07:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-7ebb90-predictor-6b8ccc595-6qcz5 namespace: kserve-ci-e2e-test resourceVersion: "18846" uid: ca4c3af6-5616-42b6-9b89-ee283f3cf89b kind: Event lastTimestamp: "2026-04-16T17:07:43Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T17:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:07:43Z" name: isvc-init-fail-7ebb90-predictor-6b8ccc595-6qcz5.18a6e555c9a43c97 namespace: kserve-ci-e2e-test resourceVersion: "18918" uid: f5bfb2b6-d287-409a-bcf8-5e81b67cb1fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-7ebb90-predictor-6b8ccc595-6qcz5 namespace: kserve-ci-e2e-test resourceVersion: "18846" uid: ca4c3af6-5616-42b6-9b89-ee283f3cf89b kind: Event lastTimestamp: "2026-04-16T17:07:44Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-04-16T17:07:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:07:44Z" name: isvc-init-fail-7ebb90-predictor-6b8ccc595-6qcz5.18a6e556da87f3f2 namespace: kserve-ci-e2e-test resourceVersion: "18965" uid: 113226f8-f9d3-40b8-bd7d-9b39cf4d584c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-7ebb90-predictor-6b8ccc595 namespace: kserve-ci-e2e-test resourceVersion: "18835" uid: 928f440b-791d-4b99-ae61-832d3bb2e67b kind: Event lastTimestamp: "2026-04-16T17:07:39Z" message: 'Created pod: isvc-init-fail-7ebb90-predictor-6b8ccc595-6qcz5' metadata: creationTimestamp: "2026-04-16T17:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:07:39Z" name: isvc-init-fail-7ebb90-predictor-6b8ccc595.18a6e555aa714e80 namespace: kserve-ci-e2e-test resourceVersion: "18845" uid: 9e7b003d-85ea-4588-a7be-c75418f3ab15 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-7ebb90-predictor namespace: kserve-ci-e2e-test resourceVersion: "18834" uid: 9ffc8b94-7862-4d2a-ae48-f3ca338ba2db kind: Event lastTimestamp: "2026-04-16T17:07:39Z" message: Scaled up replica set isvc-init-fail-7ebb90-predictor-6b8ccc595 from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:07:39Z" name: isvc-init-fail-7ebb90-predictor.18a6e555a7b15a99 namespace: kserve-ci-e2e-test resourceVersion: "18837" uid: 8b60aebb-2f83-4ac8-8301-33b73b15c82b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-16T17:07:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-7ebb90 namespace: kserve-ci-e2e-test resourceVersion: "18820" uid: 39116449-52cf-4f39-9b38-7796d7a1d196 kind: Event lastTimestamp: "2026-04-16T17:07:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T17:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:07:43Z" name: isvc-init-fail-7ebb90.18a6e555a7c802d2 namespace: kserve-ci-e2e-test resourceVersion: "18919" uid: 84a0bfb1-fbc7-44f7-9fda-bff53fb9c856 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:07:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-7ebb90 namespace: kserve-ci-e2e-test resourceVersion: "18820" uid: 39116449-52cf-4f39-9b38-7796d7a1d196 kind: Event lastTimestamp: "2026-04-16T17:07:39Z" message: 'Failed to update status for InferenceService "isvc-init-fail-7ebb90": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-7ebb90": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:07:39Z" name: isvc-init-fail-7ebb90.18a6e555a8338c4e namespace: kserve-ci-e2e-test resourceVersion: "18857" uid: babfb69e-504f-4057-8676-f6af01bf67b5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:07:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-7ebb90 namespace: kserve-ci-e2e-test resourceVersion: "18820" uid: 39116449-52cf-4f39-9b38-7796d7a1d196 kind: Event lastTimestamp: "2026-04-16T17:07:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-7ebb90": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:07:39Z" name: isvc-init-fail-7ebb90.18a6e555a833d8fe namespace: kserve-ci-e2e-test resourceVersion: "18858" uid: b76098a0-d231-4d80-92cb-f7074afb572d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:02:40.345382Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj namespace: kserve-ci-e2e-test resourceVersion: "16379" uid: 2086a436-bded-4cab-b596-ed5ffdade66e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj to ip-10-0-130-1.ec2.internal metadata: creationTimestamp: "2026-04-16T17:02:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:02:40Z" name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj.18a6e51018f8173d namespace: kserve-ci-e2e-test resourceVersion: "16383" uid: 70ea55e3-faa0-42ca-ad25-669d4c2c0204 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f7c99d688-jwz56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj namespace: kserve-ci-e2e-test resourceVersion: "16387" uid: 2086a436-bded-4cab-b596-ed5ffdade66e kind: Event lastTimestamp: "2026-04-16T17:02:40Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:02:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:02:40Z" name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj.18a6e51031e989be namespace: kserve-ci-e2e-test resourceVersion: "16398" uid: 894aa98d-5c7a-4426-9fe3-69587ceb3b1b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 2086a436-bded-4cab-b596-ed5ffdade66e kind: Event lastTimestamp: "2026-04-16T17:02:40Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ab565fe3972fd5cd2929b41b0a6a9face626513b818fecb7a38edcdfe0857061" already present on machine metadata: creationTimestamp: "2026-04-16T17:02:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:40Z" name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj.18a6e510333328be namespace: kserve-ci-e2e-test resourceVersion: "16401" uid: ad1eacbb-4ab8-4d9a-811b-58c96bb9e98c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 2086a436-bded-4cab-b596-ed5ffdade66e kind: Event lastTimestamp: "2026-04-16T17:02:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T17:02:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:40Z" name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj.18a6e510361f42fd namespace: kserve-ci-e2e-test resourceVersion: "16402" uid: b30965dd-d98d-440f-9d59-2d869eabd3bb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 2086a436-bded-4cab-b596-ed5ffdade66e kind: Event lastTimestamp: "2026-04-16T17:02:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T17:02:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:40Z" name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj.18a6e5103680416f namespace: kserve-ci-e2e-test resourceVersion: "16403" uid: 9bafb6c1-06d7-4c50-a946-64cea60247f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 2086a436-bded-4cab-b596-ed5ffdade66e kind: Event lastTimestamp: "2026-04-16T17:02:45Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-16T17:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:45Z" name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj.18a6e5113cda3831 namespace: kserve-ci-e2e-test resourceVersion: "16442" uid: 91c62b27-4079-41ad-978a-fed27a398ef1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 2086a436-bded-4cab-b596-ed5ffdade66e kind: Event lastTimestamp: "2026-04-16T17:02:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T17:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:45Z" name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj.18a6e5113fca450e namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: d00aff14-9342-48db-aafd-316738bdb770 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 2086a436-bded-4cab-b596-ed5ffdade66e kind: Event lastTimestamp: "2026-04-16T17:02:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T17:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:45Z" name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj.18a6e5114037dac6 namespace: kserve-ci-e2e-test resourceVersion: "16450" uid: e410a4dc-89e6-4b8d-8180-2df9ce5ea09f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 2086a436-bded-4cab-b596-ed5ffdade66e kind: Event lastTimestamp: "2026-04-16T17:02:45Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:f8247bac087519464bb09f0d4b8b2d3ccc7707b40f660b20ee5a2ff256216fcd" already present on machine metadata: creationTimestamp: "2026-04-16T17:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:45Z" name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj.18a6e51140439280 namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: d5b5d7b5-a0fd-48a7-a491-224ad1d47dc4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 2086a436-bded-4cab-b596-ed5ffdade66e kind: Event lastTimestamp: "2026-04-16T17:02:45Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-16T17:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:45Z" name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj.18a6e511439c6038 namespace: kserve-ci-e2e-test resourceVersion: "16453" uid: 0930111a-26c1-4b95-a98e-c14b9a8c3c90 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 2086a436-bded-4cab-b596-ed5ffdade66e kind: Event lastTimestamp: "2026-04-16T17:02:45Z" message: Started container agent metadata: creationTimestamp: "2026-04-16T17:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:45Z" name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj.18a6e51144042ef7 namespace: kserve-ci-e2e-test resourceVersion: "16454" uid: 6d505a28-5d28-4fff-a6b2-047bbb4637d7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T17:02:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 2086a436-bded-4cab-b596-ed5ffdade66e kind: Event lastTimestamp: "2026-04-16T17:03:38Z" message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T17:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:03:38Z" name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj.18a6e51178c851b1 namespace: kserve-ci-e2e-test resourceVersion: "16854" uid: c26e204e-e815-4264-8d6d-c561be055156 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T17:02:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 2086a436-bded-4cab-b596-ed5ffdade66e kind: Event lastTimestamp: "2026-04-16T17:03:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T17:02:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:03:38Z" name: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj.18a6e511b48718bb namespace: kserve-ci-e2e-test resourceVersion: "16855" uid: 766c4ed3-e848-407b-bfd4-b00c6e878d63 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-dc743-predictor-94bcc8d69 namespace: kserve-ci-e2e-test resourceVersion: "16366" uid: a5afad3b-1d37-44a1-8fad-ece7f2618858 kind: Event lastTimestamp: "2026-04-16T17:02:40Z" message: 'Created pod: isvc-logger-raw-dc743-predictor-94bcc8d69-6pdkj' metadata: creationTimestamp: "2026-04-16T17:02:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:02:40Z" name: isvc-logger-raw-dc743-predictor-94bcc8d69.18a6e510188fece5 namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: 9902645f-795a-4414-b52b-5fa4508a6682 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-dc743-predictor namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 7591c7d7-7efa-4937-91de-28b092bd124a kind: Event lastTimestamp: "2026-04-16T17:02:40Z" message: Scaled up replica set isvc-logger-raw-dc743-predictor-94bcc8d69 from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:02:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:02:40Z" name: isvc-logger-raw-dc743-predictor.18a6e51015ad4977 namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: 7e62d113-a467-44cb-9da1-43f49118d47c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:02:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-dc743-predictor namespace: kserve-ci-e2e-test resourceVersion: "16365" uid: af61eb9c-976c-4125-ae35-1a6362210c4a kind: Event lastTimestamp: "2026-04-16T17:03:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:02:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:03:25Z" name: isvc-logger-raw-dc743-predictor.18a6e513947da190 namespace: kserve-ci-e2e-test resourceVersion: "16765" uid: 47a8f19b-bbe1-4719-b94c-ada5fc3e8a02 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:02:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-dc743-predictor namespace: kserve-ci-e2e-test resourceVersion: "16365" uid: af61eb9c-976c-4125-ae35-1a6362210c4a kind: Event lastTimestamp: "2026-04-16T17:03:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:02:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:03:25Z" name: isvc-logger-raw-dc743-predictor.18a6e51394e50315 namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: 10458ed7-5955-486c-b010-b225395b0c0d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T17:03:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-dc743-predictor namespace: kserve-ci-e2e-test resourceVersion: "16535" uid: af61eb9c-976c-4125-ae35-1a6362210c4a kind: Event lastTimestamp: "2026-04-16T17:04:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:03:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:04:40Z" name: isvc-logger-raw-dc743-predictor.18a6e517139bff46 namespace: kserve-ci-e2e-test resourceVersion: "17372" uid: e06706d5-9534-4f5a-b7c4-d95d914eb17d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T17:03:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-dc743-predictor namespace: kserve-ci-e2e-test resourceVersion: "16535" uid: af61eb9c-976c-4125-ae35-1a6362210c4a kind: Event lastTimestamp: "2026-04-16T17:04:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:03:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:04:40Z" name: isvc-logger-raw-dc743-predictor.18a6e51713fa4b08 namespace: kserve-ci-e2e-test resourceVersion: "17373" uid: e7189670-0c35-43be-8b18-208d3c0294fd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-04-16T17:02:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-dc743 namespace: kserve-ci-e2e-test resourceVersion: "16356" uid: a4d6a051-8164-472a-9a47-f8fa43c47151 kind: Event lastTimestamp: "2026-04-16T17:03:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T17:02:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:03:48Z" name: isvc-logger-raw-dc743.18a6e5101587c8a8 namespace: kserve-ci-e2e-test resourceVersion: "16940" uid: 30e43abd-4ecb-4a4e-a24c-0f18cf647a8e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:02:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-dc743 namespace: kserve-ci-e2e-test resourceVersion: "16356" uid: a4d6a051-8164-472a-9a47-f8fa43c47151 kind: Event lastTimestamp: "2026-04-16T17:02:40Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-dc743": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-dc743": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:02:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:02:40Z" name: isvc-logger-raw-dc743.18a6e51015df06a4 namespace: kserve-ci-e2e-test resourceVersion: "16389" uid: b473360b-f5ea-473c-800a-8702d2786a91 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:02:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-dc743 namespace: kserve-ci-e2e-test resourceVersion: "16356" uid: a4d6a051-8164-472a-9a47-f8fa43c47151 kind: Event lastTimestamp: "2026-04-16T17:02:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-dc743": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:02:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:02:40Z" name: isvc-logger-raw-dc743.18a6e51015df4b0f namespace: kserve-ci-e2e-test resourceVersion: "16390" uid: 87c71cd9-95db-44bd-bdd3-514d3e3bcbca reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-dc743 namespace: kserve-ci-e2e-test resourceVersion: "16938" uid: a4d6a051-8164-472a-9a47-f8fa43c47151 kind: Event lastTimestamp: "2026-04-16T17:03:48Z" message: InferenceService [isvc-logger-raw-dc743] is Ready metadata: creationTimestamp: "2026-04-16T17:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:03:48Z" name: isvc-logger-raw-dc743.18a6e51feee6c9db namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: e41e3023-c5b8-4d6f-9db3-c9a6abaa25dc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:06:00.815909Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f namespace: kserve-ci-e2e-test resourceVersion: "17973" uid: 53f05150-500a-4d35-8473-a1c6ae6ae157 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-69a2b2-predictor-56f89765c8-tzp9f to ip-10-0-130-1.ec2.internal metadata: creationTimestamp: "2026-04-16T17:06:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:06:00Z" name: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f.18a6e53ec5f189a7 namespace: kserve-ci-e2e-test resourceVersion: "17978" uid: 9aaedf5a-d62a-4c4f-808c-6588d0583e59 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f7c99d688-jwz56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f namespace: kserve-ci-e2e-test resourceVersion: "17982" uid: 53f05150-500a-4d35-8473-a1c6ae6ae157 kind: Event lastTimestamp: "2026-04-16T17:06:01Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:06:01Z" name: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f.18a6e53edf1ae8f0 namespace: kserve-ci-e2e-test resourceVersion: "17992" uid: 86f1ec61-e83c-43aa-92bb-b9eacf8f03ed reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f namespace: kserve-ci-e2e-test resourceVersion: "17975" uid: 53f05150-500a-4d35-8473-a1c6ae6ae157 kind: Event lastTimestamp: "2026-04-16T17:06:01Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ab565fe3972fd5cd2929b41b0a6a9face626513b818fecb7a38edcdfe0857061" already present on machine metadata: creationTimestamp: "2026-04-16T17:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:06:01Z" name: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f.18a6e53ee08c0438 namespace: kserve-ci-e2e-test resourceVersion: "17994" uid: 1e9d039c-252a-4c4c-a4e4-39ccd6e1866b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f namespace: kserve-ci-e2e-test resourceVersion: "17975" uid: 53f05150-500a-4d35-8473-a1c6ae6ae157 kind: Event lastTimestamp: "2026-04-16T17:06:01Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T17:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:06:01Z" name: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f.18a6e53ee3f31ee8 namespace: kserve-ci-e2e-test resourceVersion: "17995" uid: 002d491b-e522-445f-a021-23e4c9641722 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f namespace: kserve-ci-e2e-test resourceVersion: "17975" uid: 53f05150-500a-4d35-8473-a1c6ae6ae157 kind: Event lastTimestamp: "2026-04-16T17:06:01Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T17:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:06:01Z" name: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f.18a6e53ee46fe448 namespace: kserve-ci-e2e-test resourceVersion: "17996" uid: d8800fff-d35c-4287-a71c-3cd4d6fe9652 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f namespace: kserve-ci-e2e-test resourceVersion: "17975" uid: 53f05150-500a-4d35-8473-a1c6ae6ae157 kind: Event lastTimestamp: "2026-04-16T17:06:05Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-16T17:06:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:06:05Z" name: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f.18a6e53ff43a3750 namespace: kserve-ci-e2e-test resourceVersion: "18034" uid: 181a050d-05d9-46cc-92a4-1b4565bd6442 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f namespace: kserve-ci-e2e-test resourceVersion: "17975" uid: 53f05150-500a-4d35-8473-a1c6ae6ae157 kind: Event lastTimestamp: "2026-04-16T17:06:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T17:06:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:06:05Z" name: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f.18a6e53ff84d93c0 namespace: kserve-ci-e2e-test resourceVersion: "18038" uid: d59493d1-4b9c-44b8-ac1a-d575b77b3fb4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f namespace: kserve-ci-e2e-test resourceVersion: "17975" uid: 53f05150-500a-4d35-8473-a1c6ae6ae157 kind: Event lastTimestamp: "2026-04-16T17:06:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T17:06:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:06:05Z" name: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f.18a6e53ff8c4d0c5 namespace: kserve-ci-e2e-test resourceVersion: "18040" uid: 86bd6ebe-fad6-4527-8105-e8c234fd5d74 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T17:06:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f namespace: kserve-ci-e2e-test resourceVersion: "17975" uid: 53f05150-500a-4d35-8473-a1c6ae6ae157 kind: Event lastTimestamp: "2026-04-16T17:07:07Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T17:06:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:07:07Z" name: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f.18a6e54030202b55 namespace: kserve-ci-e2e-test resourceVersion: "18483" uid: b741c275-bb02-4545-af8a-0fb3a8eb2f54 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f namespace: kserve-ci-e2e-test resourceVersion: "17975" uid: 53f05150-500a-4d35-8473-a1c6ae6ae157 kind: Event lastTimestamp: "2026-04-16T17:07:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T17:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:07:39Z" name: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f.18a6e555a589490d namespace: kserve-ci-e2e-test resourceVersion: "18825" uid: 099e9d25-6335-4b84-99f9-1b9371303fc1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-69a2b2-predictor-56f89765c8 namespace: kserve-ci-e2e-test resourceVersion: "17964" uid: 052a9ffb-5294-4adf-b2ad-80080412fa7c kind: Event lastTimestamp: "2026-04-16T17:06:00Z" message: 'Created pod: isvc-primary-69a2b2-predictor-56f89765c8-tzp9f' metadata: creationTimestamp: "2026-04-16T17:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:06:00Z" name: isvc-primary-69a2b2-predictor-56f89765c8.18a6e53ec5706ffb namespace: kserve-ci-e2e-test resourceVersion: "17974" uid: 3091e45d-20aa-4a13-986d-f6329ebc9503 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-69a2b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17962" uid: c57e937e-d0e2-4672-8ed6-8cdc3ee909b4 kind: Event lastTimestamp: "2026-04-16T17:06:00Z" message: Scaled up replica set isvc-primary-69a2b2-predictor-56f89765c8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:06:00Z" name: isvc-primary-69a2b2-predictor.18a6e53ec1b88527 namespace: kserve-ci-e2e-test resourceVersion: "17966" uid: c655bddd-a493-4763-85ad-9bad99f6fa94 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:06:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-69a2b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17963" uid: 6151ff8c-5e7d-44a4-9283-8548d80a12a5 kind: Event lastTimestamp: "2026-04-16T17:06:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:06:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:06:30Z" name: isvc-primary-69a2b2-predictor.18a6e54240342121 namespace: kserve-ci-e2e-test resourceVersion: "18221" uid: 2338da8c-c5cc-44f9-ab85-7fd8df3199b3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:06:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-69a2b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17963" uid: 6151ff8c-5e7d-44a4-9283-8548d80a12a5 kind: Event lastTimestamp: "2026-04-16T17:06:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:06:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:06:30Z" name: isvc-primary-69a2b2-predictor.18a6e5424093c24d namespace: kserve-ci-e2e-test resourceVersion: "18222" uid: da438894-6225-456b-88cd-65376ab8aaa5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T17:06:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-69a2b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18108" uid: 6151ff8c-5e7d-44a4-9283-8548d80a12a5 kind: Event lastTimestamp: "2026-04-16T17:07:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:07:30Z" name: isvc-primary-69a2b2-predictor.18a6e5493e029689 namespace: kserve-ci-e2e-test resourceVersion: "18723" uid: ccf540ff-70b9-465f-a60b-865d981f9da8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T17:06:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-69a2b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18108" uid: 6151ff8c-5e7d-44a4-9283-8548d80a12a5 kind: Event lastTimestamp: "2026-04-16T17:07:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:07:30Z" name: isvc-primary-69a2b2-predictor.18a6e5493e608ed0 namespace: kserve-ci-e2e-test resourceVersion: "18724" uid: 752e0144-012f-4593-85d4-0e00434d2f71 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-04-16T17:06:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-69a2b2 namespace: kserve-ci-e2e-test resourceVersion: "17954" uid: dace4eeb-40b5-4d39-8b96-2f1283a58ce2 kind: Event lastTimestamp: "2026-04-16T17:07:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T17:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:07:17Z" name: isvc-primary-69a2b2.18a6e53ec197fe97 namespace: kserve-ci-e2e-test resourceVersion: "18567" uid: f61df6fb-5232-4c82-b117-f2930a1ab601 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-69a2b2 namespace: kserve-ci-e2e-test resourceVersion: "17954" uid: dace4eeb-40b5-4d39-8b96-2f1283a58ce2 kind: Event lastTimestamp: "2026-04-16T17:06:00Z" message: 'Failed to update status for InferenceService "isvc-primary-69a2b2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-69a2b2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:06:00Z" name: isvc-primary-69a2b2.18a6e53ec1ea2f36 namespace: kserve-ci-e2e-test resourceVersion: "17968" uid: 01ca2139-50ac-4d25-a229-ffc403ff83c0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-69a2b2 namespace: kserve-ci-e2e-test resourceVersion: "17954" uid: dace4eeb-40b5-4d39-8b96-2f1283a58ce2 kind: Event lastTimestamp: "2026-04-16T17:06:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-69a2b2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:06:00Z" name: isvc-primary-69a2b2.18a6e53ec1ea773e namespace: kserve-ci-e2e-test resourceVersion: "17969" uid: c94f391f-e7a4-487f-b15f-81ab776d1c9f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-69a2b2 namespace: kserve-ci-e2e-test resourceVersion: "18565" uid: dace4eeb-40b5-4d39-8b96-2f1283a58ce2 kind: Event lastTimestamp: "2026-04-16T17:07:17Z" message: InferenceService [isvc-primary-69a2b2] is Ready metadata: creationTimestamp: "2026-04-16T17:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:07:17Z" name: isvc-primary-69a2b2.18a6e550bc1ab287 namespace: kserve-ci-e2e-test resourceVersion: "18566" uid: 0eb662d3-a03d-4714-b322-fab347e2c7c0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:57:39.415901Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw namespace: kserve-ci-e2e-test resourceVersion: "13594" uid: e3bcf0d4-3673-4f6b-a638-04809c5e5256 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw to ip-10-0-130-1.ec2.internal metadata: creationTimestamp: "2026-04-16T16:57:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:57:39Z" name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw.18a6e4ca082c9c3e namespace: kserve-ci-e2e-test resourceVersion: "13600" uid: cb0054e6-adbb-4c8c-bf6a-3fc0df518c9d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f7c99d688-jwz56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:57:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw namespace: kserve-ci-e2e-test resourceVersion: "13604" uid: e3bcf0d4-3673-4f6b-a638-04809c5e5256 kind: Event lastTimestamp: "2026-04-16T16:57:39Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:57:39Z" name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw.18a6e4ca2195305e namespace: kserve-ci-e2e-test resourceVersion: "13619" uid: ba29a3d4-b379-44b1-a7a1-f8ef7a242214 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:57:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: e3bcf0d4-3673-4f6b-a638-04809c5e5256 kind: Event lastTimestamp: "2026-04-16T16:57:39Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:ab565fe3972fd5cd2929b41b0a6a9face626513b818fecb7a38edcdfe0857061" metadata: creationTimestamp: "2026-04-16T16:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:57:39Z" name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw.18a6e4ca22fab044 namespace: kserve-ci-e2e-test resourceVersion: "13621" uid: b6695670-d93a-4ffd-861c-db48d27aa451 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:57:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: e3bcf0d4-3673-4f6b-a638-04809c5e5256 kind: Event lastTimestamp: "2026-04-16T16:57:43Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:ab565fe3972fd5cd2929b41b0a6a9face626513b818fecb7a38edcdfe0857061" in 3.318s (3.318s including waiting). Image size: 299668949 bytes.' metadata: creationTimestamp: "2026-04-16T16:57:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:57:43Z" name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw.18a6e4cae8cc0b41 namespace: kserve-ci-e2e-test resourceVersion: "13645" uid: 970c19f6-f27d-4319-96fe-1e125725db00 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:57:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: e3bcf0d4-3673-4f6b-a638-04809c5e5256 kind: Event lastTimestamp: "2026-04-16T16:57:43Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T16:57:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:57:43Z" name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw.18a6e4caebcfeefb namespace: kserve-ci-e2e-test resourceVersion: "13647" uid: c210b968-e8be-4354-9347-fccffffafd29 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:57:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: e3bcf0d4-3673-4f6b-a638-04809c5e5256 kind: Event lastTimestamp: "2026-04-16T16:57:43Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T16:57:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:57:43Z" name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw.18a6e4caec315302 namespace: kserve-ci-e2e-test resourceVersion: "13648" uid: 2a77f53f-9860-4d25-99a3-1ba2ba264920 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:57:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: e3bcf0d4-3673-4f6b-a638-04809c5e5256 kind: Event lastTimestamp: "2026-04-16T16:57:47Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" metadata: creationTimestamp: "2026-04-16T16:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:57:47Z" name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw.18a6e4cbd9b4ca2c namespace: kserve-ci-e2e-test resourceVersion: "13693" uid: e36caff6-1cbd-4144-8a3a-a0b598a3d772 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:58:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: e3bcf0d4-3673-4f6b-a638-04809c5e5256 kind: Event lastTimestamp: "2026-04-16T16:58:00Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" in 13.044s (13.044s including waiting). Image size: 1560615365 bytes.' metadata: creationTimestamp: "2026-04-16T16:58:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:58:00Z" name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw.18a6e4cee33a8f5f namespace: kserve-ci-e2e-test resourceVersion: "13786" uid: a866b803-b9a0-49bf-8fe2-8ea070a58093 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:58:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: e3bcf0d4-3673-4f6b-a638-04809c5e5256 kind: Event lastTimestamp: "2026-04-16T16:58:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:58:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:58:00Z" name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw.18a6e4cee6211ca1 namespace: kserve-ci-e2e-test resourceVersion: "13787" uid: d9eaed30-2b23-48aa-8d19-4b5de04451f5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:58:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: e3bcf0d4-3673-4f6b-a638-04809c5e5256 kind: Event lastTimestamp: "2026-04-16T16:58:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:58:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:58:00Z" name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw.18a6e4cee68a39b3 namespace: kserve-ci-e2e-test resourceVersion: "13788" uid: d2c20d54-357a-4dc6-b23c-c127ff0e576f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:58:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: e3bcf0d4-3673-4f6b-a638-04809c5e5256 kind: Event lastTimestamp: "2026-04-16T16:58:00Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:f8247bac087519464bb09f0d4b8b2d3ccc7707b40f660b20ee5a2ff256216fcd" metadata: creationTimestamp: "2026-04-16T16:58:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:58:00Z" name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw.18a6e4cee69640a7 namespace: kserve-ci-e2e-test resourceVersion: "13789" uid: a1c3008a-c7a8-4dff-b6ef-8d0a12f36c76 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:58:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: e3bcf0d4-3673-4f6b-a638-04809c5e5256 kind: Event lastTimestamp: "2026-04-16T16:58:02Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:f8247bac087519464bb09f0d4b8b2d3ccc7707b40f660b20ee5a2ff256216fcd" in 2.039s (2.039s including waiting). Image size: 237893238 bytes.' metadata: creationTimestamp: "2026-04-16T16:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:58:02Z" name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw.18a6e4cf6025899e namespace: kserve-ci-e2e-test resourceVersion: "13802" uid: acbf6797-5253-4faa-9630-91696d137af8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:58:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: e3bcf0d4-3673-4f6b-a638-04809c5e5256 kind: Event lastTimestamp: "2026-04-16T16:58:02Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-16T16:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:58:02Z" name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw.18a6e4cf63058958 namespace: kserve-ci-e2e-test resourceVersion: "13804" uid: 6a302917-8ae1-4ee2-9616-33c8cd659113 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:58:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: e3bcf0d4-3673-4f6b-a638-04809c5e5256 kind: Event lastTimestamp: "2026-04-16T16:58:02Z" message: Started container agent metadata: creationTimestamp: "2026-04-16T16:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:58:02Z" name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw.18a6e4cf63663dab namespace: kserve-ci-e2e-test resourceVersion: "13805" uid: ae7035c5-ca9c-4ed5-988a-5802ed8aa8e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T16:58:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: e3bcf0d4-3673-4f6b-a638-04809c5e5256 kind: Event lastTimestamp: "2026-04-16T16:58:54Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:58:54Z" name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw.18a6e4cf97409a27 namespace: kserve-ci-e2e-test resourceVersion: "14176" uid: 7e70dff1-2248-4be4-9b0f-4baad8e145b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T16:58:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: e3bcf0d4-3673-4f6b-a638-04809c5e5256 kind: Event lastTimestamp: "2026-04-16T16:58:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T16:58:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:58:54Z" name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw.18a6e4cf974a3e3d namespace: kserve-ci-e2e-test resourceVersion: "14177" uid: a2184121-40ad-40d4-9d88-0bdac09ab040 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:57:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67 namespace: kserve-ci-e2e-test resourceVersion: "13584" uid: a9e3bde0-3a30-40b9-91ec-af3f95b03b0f kind: Event lastTimestamp: "2026-04-16T16:57:39Z" message: 'Created pod: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67-kxgcw' metadata: creationTimestamp: "2026-04-16T16:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:57:39Z" name: isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67.18a6e4ca07c79f82 namespace: kserve-ci-e2e-test resourceVersion: "13596" uid: 62694219-ff74-4f72-a57c-f0bcd5bc3bd4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:57:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-4801b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13583" uid: 29c69f63-2830-410e-bd92-d3c50162828c kind: Event lastTimestamp: "2026-04-16T16:57:39Z" message: Scaled up replica set isvc-raw-sklearn-batcher-4801b-predictor-78465c5d67 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:57:39Z" name: isvc-raw-sklearn-batcher-4801b-predictor.18a6e4ca011b7677 namespace: kserve-ci-e2e-test resourceVersion: "13585" uid: 4a75241f-b86c-4f83-86e0-ce3e71d6b17d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:57:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-4801b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13592" uid: 9f9675d0-d7f4-40f1-b9d6-fad226f4108a kind: Event lastTimestamp: "2026-04-16T16:58:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:57:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:58:39Z" name: isvc-raw-sklearn-batcher-4801b-predictor.18a6e4cd866a0517 namespace: kserve-ci-e2e-test resourceVersion: "14078" uid: 1657d162-c369-4731-b959-d33d1c5a0dbf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:57:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-4801b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13592" uid: 9f9675d0-d7f4-40f1-b9d6-fad226f4108a kind: Event lastTimestamp: "2026-04-16T16:58:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:57:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:58:39Z" name: isvc-raw-sklearn-batcher-4801b-predictor.18a6e4cd86d17290 namespace: kserve-ci-e2e-test resourceVersion: "14079" uid: 48face3c-fdde-4bd3-9580-5f8a2072c251 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:58:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-4801b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13741" uid: 9f9675d0-d7f4-40f1-b9d6-fad226f4108a kind: Event lastTimestamp: "2026-04-16T16:59:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:58:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:59:39Z" name: isvc-raw-sklearn-batcher-4801b-predictor.18a6e4d4846fcb5d namespace: kserve-ci-e2e-test resourceVersion: "14628" uid: 90daea0e-914a-4fb4-97c3-a1e2c621216b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:58:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-4801b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13741" uid: 9f9675d0-d7f4-40f1-b9d6-fad226f4108a kind: Event lastTimestamp: "2026-04-16T16:59:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:58:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:59:39Z" name: isvc-raw-sklearn-batcher-4801b-predictor.18a6e4d484cf2d22 namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: e3671bb9-7a19-43dc-921a-e3873e21b04d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 15 eventTime: null firstTimestamp: "2026-04-16T16:57:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-4801b namespace: kserve-ci-e2e-test resourceVersion: "13572" uid: 062b81a4-0a2e-4e3c-b5a8-3e018aaec8f1 kind: Event lastTimestamp: "2026-04-16T16:59:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:59:04Z" name: isvc-raw-sklearn-batcher-4801b.18a6e4ca073ba9f1 namespace: kserve-ci-e2e-test resourceVersion: "14262" uid: a6535286-bdcc-44d3-8630-5a4640ff9480 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:57:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-4801b namespace: kserve-ci-e2e-test resourceVersion: "13572" uid: 062b81a4-0a2e-4e3c-b5a8-3e018aaec8f1 kind: Event lastTimestamp: "2026-04-16T16:57:39Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-4801b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-4801b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:57:39Z" name: isvc-raw-sklearn-batcher-4801b.18a6e4ca078b0f03 namespace: kserve-ci-e2e-test resourceVersion: "13609" uid: f518a731-b6af-4326-95f9-8b26c658aaac reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:57:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-4801b namespace: kserve-ci-e2e-test resourceVersion: "13572" uid: 062b81a4-0a2e-4e3c-b5a8-3e018aaec8f1 kind: Event lastTimestamp: "2026-04-16T16:57:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-4801b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:57:39Z" name: isvc-raw-sklearn-batcher-4801b.18a6e4ca078b686b namespace: kserve-ci-e2e-test resourceVersion: "13610" uid: eaf0fdc5-c48d-4c34-9223-45d9527a2517 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-4801b namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: 062b81a4-0a2e-4e3c-b5a8-3e018aaec8f1 kind: Event lastTimestamp: "2026-04-16T16:59:04Z" message: InferenceService [isvc-raw-sklearn-batcher-4801b] is Ready metadata: creationTimestamp: "2026-04-16T16:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:59:04Z" name: isvc-raw-sklearn-batcher-4801b.18a6e4ddcfe0cecc namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: ebaf5d60-59dd-4fa2-b709-e520c8baac75 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:07:20.978184Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-69a2b2-predictor-bc7cfb67-f5tl8 namespace: kserve-ci-e2e-test resourceVersion: "18616" uid: c5e0a32f-691b-4e70-98b2-50a42b061857 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-69a2b2-predictor-bc7cfb67-f5tl8 to ip-10-0-130-1.ec2.internal metadata: creationTimestamp: "2026-04-16T17:07:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:07:20Z" name: isvc-secondary-69a2b2-predictor-bc7cfb67-f5tl8.18a6e5516ffce147 namespace: kserve-ci-e2e-test resourceVersion: "18623" uid: d65552c7-0d8a-4a70-8500-809998c31713 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f7c99d688-jwz56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:21Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-69a2b2-predictor-bc7cfb67-f5tl8 namespace: kserve-ci-e2e-test resourceVersion: "18626" uid: c5e0a32f-691b-4e70-98b2-50a42b061857 kind: Event lastTimestamp: "2026-04-16T17:07:21Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:07:21Z" name: isvc-secondary-69a2b2-predictor-bc7cfb67-f5tl8.18a6e55189c2807c namespace: kserve-ci-e2e-test resourceVersion: "18632" uid: 235ccc9a-f14f-4ef3-8aab-b31942d0ee56 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:07:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-69a2b2-predictor-bc7cfb67-f5tl8 namespace: kserve-ci-e2e-test resourceVersion: "18619" uid: c5e0a32f-691b-4e70-98b2-50a42b061857 kind: Event lastTimestamp: "2026-04-16T17:07:27Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ab565fe3972fd5cd2929b41b0a6a9face626513b818fecb7a38edcdfe0857061" already present on machine metadata: creationTimestamp: "2026-04-16T17:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:07:27Z" name: isvc-secondary-69a2b2-predictor-bc7cfb67-f5tl8.18a6e5518b0006fc namespace: kserve-ci-e2e-test resourceVersion: "18683" uid: 2944e777-164b-4a34-b50f-89a2c264193a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:07:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-69a2b2-predictor-bc7cfb67-f5tl8 namespace: kserve-ci-e2e-test resourceVersion: "18619" uid: c5e0a32f-691b-4e70-98b2-50a42b061857 kind: Event lastTimestamp: "2026-04-16T17:07:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T17:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:07:27Z" name: isvc-secondary-69a2b2-predictor-bc7cfb67-f5tl8.18a6e5518eac7281 namespace: kserve-ci-e2e-test resourceVersion: "18689" uid: f364c4c6-bf1e-4fda-9c06-5ff837e863b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:07:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-69a2b2-predictor-bc7cfb67-f5tl8 namespace: kserve-ci-e2e-test resourceVersion: "18619" uid: c5e0a32f-691b-4e70-98b2-50a42b061857 kind: Event lastTimestamp: "2026-04-16T17:07:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T17:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:07:27Z" name: isvc-secondary-69a2b2-predictor-bc7cfb67-f5tl8.18a6e5518f278939 namespace: kserve-ci-e2e-test resourceVersion: "18690" uid: a2dd9400-9dd8-4178-a927-5004f6aad875 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-69a2b2-predictor-bc7cfb67-f5tl8 namespace: kserve-ci-e2e-test resourceVersion: "18619" uid: c5e0a32f-691b-4e70-98b2-50a42b061857 kind: Event lastTimestamp: "2026-04-16T17:07:30Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-69a2b2-predictor-bc7cfb67-f5tl8_kserve-ci-e2e-test(c5e0a32f-691b-4e70-98b2-50a42b061857) metadata: creationTimestamp: "2026-04-16T17:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:07:30Z" name: isvc-secondary-69a2b2-predictor-bc7cfb67-f5tl8.18a6e55394ea8407 namespace: kserve-ci-e2e-test resourceVersion: "18712" uid: f4306fea-c408-412a-a374-6b706c55e282 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-69a2b2-predictor-bc7cfb67 namespace: kserve-ci-e2e-test resourceVersion: "18605" uid: 8332bdfe-6b26-4d02-b515-91630980e77f kind: Event lastTimestamp: "2026-04-16T17:07:20Z" message: 'Created pod: isvc-secondary-69a2b2-predictor-bc7cfb67-f5tl8' metadata: creationTimestamp: "2026-04-16T17:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:07:20Z" name: isvc-secondary-69a2b2-predictor-bc7cfb67.18a6e5516f49f109 namespace: kserve-ci-e2e-test resourceVersion: "18620" uid: 663cf0d6-e31b-4258-a2d7-8fee6b7c92ac reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-69a2b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18603" uid: f49f0e43-e662-4220-ac20-d8ba13d8d137 kind: Event lastTimestamp: "2026-04-16T17:07:20Z" message: Scaled up replica set isvc-secondary-69a2b2-predictor-bc7cfb67 from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:07:20Z" name: isvc-secondary-69a2b2-predictor.18a6e5516c580318 namespace: kserve-ci-e2e-test resourceVersion: "18607" uid: 06a97330-205e-4b07-8ab3-1af2b6f6e318 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-69a2b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18604" uid: 065e40af-1c0c-4ae9-8ac3-8baf05fe38fe kind: Event lastTimestamp: "2026-04-16T17:07:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:07:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:07:35Z" name: isvc-secondary-69a2b2-predictor.18a6e554eabf7848 namespace: kserve-ci-e2e-test resourceVersion: "18759" uid: 47bc108b-ce67-4960-8e9c-8f2c195c884a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-69a2b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18604" uid: 065e40af-1c0c-4ae9-8ac3-8baf05fe38fe kind: Event lastTimestamp: "2026-04-16T17:07:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:07:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:07:35Z" name: isvc-secondary-69a2b2-predictor.18a6e554eb21d802 namespace: kserve-ci-e2e-test resourceVersion: "18761" uid: ba1cdb33-f6ed-489f-ba72-2a5622f23a66 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T17:07:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-69a2b2 namespace: kserve-ci-e2e-test resourceVersion: "18595" uid: 068175b1-e741-4b54-b1b4-5ea7797a946a kind: Event lastTimestamp: "2026-04-16T17:07:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T17:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:07:30Z" name: isvc-secondary-69a2b2.18a6e5516c2bd167 namespace: kserve-ci-e2e-test resourceVersion: "18725" uid: 42106e5f-dcc1-4feb-9edb-92a82f00f049 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-69a2b2 namespace: kserve-ci-e2e-test resourceVersion: "18595" uid: 068175b1-e741-4b54-b1b4-5ea7797a946a kind: Event lastTimestamp: "2026-04-16T17:07:20Z" message: 'Failed to update status for InferenceService "isvc-secondary-69a2b2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-69a2b2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:07:20Z" name: isvc-secondary-69a2b2.18a6e5516c79a51f namespace: kserve-ci-e2e-test resourceVersion: "18609" uid: e470c6c3-6623-48b9-af7b-51f01ca49ce8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-69a2b2 namespace: kserve-ci-e2e-test resourceVersion: "18595" uid: 068175b1-e741-4b54-b1b4-5ea7797a946a kind: Event lastTimestamp: "2026-04-16T17:07:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-69a2b2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:07:20Z" name: isvc-secondary-69a2b2.18a6e5516c79ee5a namespace: kserve-ci-e2e-test resourceVersion: "18610" uid: 7355679f-472d-45b4-b21d-a9b13808518e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:59:09.764504Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc namespace: kserve-ci-e2e-test resourceVersion: "14335" uid: 73978eeb-823c-408c-9097-74093ed71133 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc to ip-10-0-130-1.ec2.internal metadata: creationTimestamp: "2026-04-16T16:59:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:59:09Z" name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc.18a6e4df115ef675 namespace: kserve-ci-e2e-test resourceVersion: "14345" uid: 73827a8c-68cc-4fc8-bea9-0e42fa9db845 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f7c99d688-jwz56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc namespace: kserve-ci-e2e-test resourceVersion: "14350" uid: 73978eeb-823c-408c-9097-74093ed71133 kind: Event lastTimestamp: "2026-04-16T16:59:10Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:59:10Z" name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc.18a6e4df2b155092 namespace: kserve-ci-e2e-test resourceVersion: "14373" uid: 1e9667f6-7ce4-48c7-ad8f-9d61eff41b1b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc namespace: kserve-ci-e2e-test resourceVersion: "14340" uid: 73978eeb-823c-408c-9097-74093ed71133 kind: Event lastTimestamp: "2026-04-16T16:59:10Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ab565fe3972fd5cd2929b41b0a6a9face626513b818fecb7a38edcdfe0857061" already present on machine metadata: creationTimestamp: "2026-04-16T16:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:59:10Z" name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc.18a6e4df2d70ddaa namespace: kserve-ci-e2e-test resourceVersion: "14376" uid: 568ee91e-4c7c-4989-8ce0-18b34e763e0a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc namespace: kserve-ci-e2e-test resourceVersion: "14340" uid: 73978eeb-823c-408c-9097-74093ed71133 kind: Event lastTimestamp: "2026-04-16T16:59:10Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T16:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:59:10Z" name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc.18a6e4df31518381 namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: 74c5f184-5871-4e37-b415-f2de016e4ae2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc namespace: kserve-ci-e2e-test resourceVersion: "14340" uid: 73978eeb-823c-408c-9097-74093ed71133 kind: Event lastTimestamp: "2026-04-16T16:59:10Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T16:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:59:10Z" name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc.18a6e4df31cd2603 namespace: kserve-ci-e2e-test resourceVersion: "14384" uid: a95e0e1c-28de-47d5-a90e-2875b0d5fd61 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc namespace: kserve-ci-e2e-test resourceVersion: "14340" uid: 73978eeb-823c-408c-9097-74093ed71133 kind: Event lastTimestamp: "2026-04-16T16:59:14Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-16T16:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:59:14Z" name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc.18a6e4e02bdd764b namespace: kserve-ci-e2e-test resourceVersion: "14426" uid: ba39c00b-d000-4fca-8179-22a435e9b5ff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc namespace: kserve-ci-e2e-test resourceVersion: "14340" uid: 73978eeb-823c-408c-9097-74093ed71133 kind: Event lastTimestamp: "2026-04-16T16:59:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:59:14Z" name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc.18a6e4e02effa130 namespace: kserve-ci-e2e-test resourceVersion: "14432" uid: 28b309bb-cf68-4ef9-90db-f9067e465a2f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc namespace: kserve-ci-e2e-test resourceVersion: "14340" uid: 73978eeb-823c-408c-9097-74093ed71133 kind: Event lastTimestamp: "2026-04-16T16:59:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:59:14Z" name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc.18a6e4e02f7b5eab namespace: kserve-ci-e2e-test resourceVersion: "14433" uid: 926969b8-dcb5-4e20-933c-af9a712a6725 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T16:59:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc namespace: kserve-ci-e2e-test resourceVersion: "14340" uid: 73978eeb-823c-408c-9097-74093ed71133 kind: Event lastTimestamp: "2026-04-16T17:00:16Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:59:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:16Z" name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc.18a6e4e0680945b6 namespace: kserve-ci-e2e-test resourceVersion: "14904" uid: e4cae889-9afc-4cf7-8ce3-43deb45c1cae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc namespace: kserve-ci-e2e-test resourceVersion: "14340" uid: 73978eeb-823c-408c-9097-74093ed71133 kind: Event lastTimestamp: "2026-04-16T17:00:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T17:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:49Z" name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc.18a6e4f666fff99e namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: 11ff3a41-3ab9-4014-adab-b3ee1121754d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697 namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: 55c1a48d-679e-47cb-8b29-1777c4958b52 kind: Event lastTimestamp: "2026-04-16T16:59:09Z" message: 'Created pod: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697-79khc' metadata: creationTimestamp: "2026-04-16T16:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:59:09Z" name: isvc-sklearn-graph-raw-231d1-predictor-5487fb9697.18a6e4df10d6a2ee namespace: kserve-ci-e2e-test resourceVersion: "14338" uid: e9a2dbd6-2f6a-4a26-b7c1-930f4d9c9cc1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-231d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: e6181831-2acc-4071-b1b1-f2d747b6e1c1 kind: Event lastTimestamp: "2026-04-16T16:59:09Z" message: Scaled up replica set isvc-sklearn-graph-raw-231d1-predictor-5487fb9697 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:59:09Z" name: isvc-sklearn-graph-raw-231d1-predictor.18a6e4df0c3e3bd5 namespace: kserve-ci-e2e-test resourceVersion: "14320" uid: b1766f04-aa7a-4c5f-a951-237e9a08e901 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-231d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 560ed488-e12f-4d5b-b374-d1bc82d25408 kind: Event lastTimestamp: "2026-04-16T16:59:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:59:24Z" name: isvc-sklearn-graph-raw-231d1-predictor.18a6e4e28a925c8f namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: 8f1ca779-0c5c-48b4-a2d8-c7f228f6e02c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-231d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 560ed488-e12f-4d5b-b374-d1bc82d25408 kind: Event lastTimestamp: "2026-04-16T16:59:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:59:24Z" name: isvc-sklearn-graph-raw-231d1-predictor.18a6e4e28aed2b7c namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: 11c430fb-0525-43a8-acb7-bfb1598084e4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:59:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-231d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14511" uid: 560ed488-e12f-4d5b-b374-d1bc82d25408 kind: Event lastTimestamp: "2026-04-16T17:00:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:00:39Z" name: isvc-sklearn-graph-raw-231d1-predictor.18a6e4e6098cd61a namespace: kserve-ci-e2e-test resourceVersion: "15102" uid: 504f0cd8-b1d6-4744-a4fc-466f9c77822c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:59:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-231d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14511" uid: 560ed488-e12f-4d5b-b374-d1bc82d25408 kind: Event lastTimestamp: "2026-04-16T17:00:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:00:39Z" name: isvc-sklearn-graph-raw-231d1-predictor.18a6e4e609f1237b namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 4d8c06c0-5a7b-4044-b575-5e45c641527c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T16:59:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-231d1 namespace: kserve-ci-e2e-test resourceVersion: "14304" uid: ac0e5c36-6d1b-4d5a-9912-4917b85481d2 kind: Event lastTimestamp: "2026-04-16T17:00:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:00:26Z" name: isvc-sklearn-graph-raw-231d1.18a6e4df0bf3580b namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: 052c7a89-7863-49f9-b6bc-4b3c39c8a9a2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-231d1 namespace: kserve-ci-e2e-test resourceVersion: "14990" uid: ac0e5c36-6d1b-4d5a-9912-4917b85481d2 kind: Event lastTimestamp: "2026-04-16T17:00:26Z" message: InferenceService [isvc-sklearn-graph-raw-231d1] is Ready metadata: creationTimestamp: "2026-04-16T17:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:00:26Z" name: isvc-sklearn-graph-raw-231d1.18a6e4f0f4023e53 namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: 3b7e1bc2-77be-4671-8e4b-a66c9b7cb968 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:00:50.030234Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4 namespace: kserve-ci-e2e-test resourceVersion: "15275" uid: 884f570a-0fa7-4098-b34f-91043f2bfce3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4 to ip-10-0-130-1.ec2.internal metadata: creationTimestamp: "2026-04-16T17:00:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:00:50Z" name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4.18a6e4f669acc236 namespace: kserve-ci-e2e-test resourceVersion: "15284" uid: 8cccb2d8-bada-40b5-8570-c99f0b7b195c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f7c99d688-jwz56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4 namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: 884f570a-0fa7-4098-b34f-91043f2bfce3 kind: Event lastTimestamp: "2026-04-16T17:00:50Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:00:50Z" name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4.18a6e4f683b99fab namespace: kserve-ci-e2e-test resourceVersion: "15341" uid: dbe8c43d-7245-4b04-862c-a47a8b60f7f1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4 namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: 884f570a-0fa7-4098-b34f-91043f2bfce3 kind: Event lastTimestamp: "2026-04-16T17:00:50Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ab565fe3972fd5cd2929b41b0a6a9face626513b818fecb7a38edcdfe0857061" already present on machine metadata: creationTimestamp: "2026-04-16T17:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:50Z" name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4.18a6e4f6852615f1 namespace: kserve-ci-e2e-test resourceVersion: "15343" uid: 49d969de-50aa-4451-b529-86e432e17f02 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4 namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: 884f570a-0fa7-4098-b34f-91043f2bfce3 kind: Event lastTimestamp: "2026-04-16T17:00:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T17:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:50Z" name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4.18a6e4f689856ecd namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: 165e7f1b-cd85-41ba-9c7e-02215d898663 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4 namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: 884f570a-0fa7-4098-b34f-91043f2bfce3 kind: Event lastTimestamp: "2026-04-16T17:00:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T17:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:50Z" name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4.18a6e4f68a027f69 namespace: kserve-ci-e2e-test resourceVersion: "15348" uid: 15343d22-3a9f-4359-8ba3-31b46c877c2c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4 namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: 884f570a-0fa7-4098-b34f-91043f2bfce3 kind: Event lastTimestamp: "2026-04-16T17:00:54Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-16T17:00:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:54Z" name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4.18a6e4f788cb4c14 namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 1bc36951-b7d9-4e1a-bac7-da1ef8a71f10 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4 namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: 884f570a-0fa7-4098-b34f-91043f2bfce3 kind: Event lastTimestamp: "2026-04-16T17:00:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T17:00:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:54Z" name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4.18a6e4f78c929e3c namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: 63020293-e8bc-4023-8155-4f7cb00041eb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4 namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: 884f570a-0fa7-4098-b34f-91043f2bfce3 kind: Event lastTimestamp: "2026-04-16T17:00:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T17:00:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:54Z" name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4.18a6e4f78d1a08ca namespace: kserve-ci-e2e-test resourceVersion: "15409" uid: 055d72aa-83a3-4d6f-961b-0e972aecaaa3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T17:00:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4 namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: 884f570a-0fa7-4098-b34f-91043f2bfce3 kind: Event lastTimestamp: "2026-04-16T17:01:56Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T17:00:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:01:56Z" name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4.18a6e4f7c5143b27 namespace: kserve-ci-e2e-test resourceVersion: "15873" uid: fa75ff27-b5b5-4484-adc9-c0b5b2a26c1a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4 namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: 884f570a-0fa7-4098-b34f-91043f2bfce3 kind: Event lastTimestamp: "2026-04-16T17:02:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T17:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:30Z" name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4.18a6e50dc056da50 namespace: kserve-ci-e2e-test resourceVersion: "16228" uid: 6cd69b44-3922-47d6-97de-a9815f5628da reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c namespace: kserve-ci-e2e-test resourceVersion: "15256" uid: cc28d0b3-2515-4fc8-aa65-a340bde1df0e kind: Event lastTimestamp: "2026-04-16T17:00:50Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c-blvm4' metadata: creationTimestamp: "2026-04-16T17:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:00:50Z" name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c.18a6e4f66922352d namespace: kserve-ci-e2e-test resourceVersion: "15278" uid: 896ead01-2b83-4b4c-b02b-0603c8af45b2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15248" uid: 760f2eaa-41a5-4253-865c-4a4deff0428d kind: Event lastTimestamp: "2026-04-16T17:00:49Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-4ab2c-predictor-75d57fbf7c from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:00:49Z" name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor.18a6e4f66590111c namespace: kserve-ci-e2e-test resourceVersion: "15259" uid: 4618e47d-a4d7-4465-a886-0c13e2b57e65 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:01:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: a3f44ab0-b19b-4c00-8e0c-f5f262086dce kind: Event lastTimestamp: "2026-04-16T17:01:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:01:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:01:04Z" name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor.18a6e4f9e480f54c namespace: kserve-ci-e2e-test resourceVersion: "15487" uid: 093bad2e-0973-45b6-8ef3-45adc4dba1e6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:01:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: a3f44ab0-b19b-4c00-8e0c-f5f262086dce kind: Event lastTimestamp: "2026-04-16T17:01:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:01:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:01:04Z" name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor.18a6e4f9e4dd89a1 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 36efdb2b-8c1e-43f1-9b6b-bbea5895b425 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T17:01:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15488" uid: a3f44ab0-b19b-4c00-8e0c-f5f262086dce kind: Event lastTimestamp: "2026-04-16T17:02:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:01:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:02:20Z" name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor.18a6e4fd63a40f55 namespace: kserve-ci-e2e-test resourceVersion: "16064" uid: 9a6c9d75-4c13-4954-892b-a8011e1e2db2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T17:01:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15488" uid: a3f44ab0-b19b-4c00-8e0c-f5f262086dce kind: Event lastTimestamp: "2026-04-16T17:02:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:02:20Z" name: isvc-sklearn-graph-raw-hpa-4ab2c-predictor.18a6e4fd6408b38c namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: d6adabf6-eab8-44b9-a480-400ca20de160 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T17:00:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-4ab2c namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: 2187e5dc-ba22-4b44-95fd-5156ea40fc7e kind: Event lastTimestamp: "2026-04-16T17:02:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T17:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:02:06Z" name: isvc-sklearn-graph-raw-hpa-4ab2c.18a6e4f6659dbbb8 namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 1b9d097a-5f98-49b5-8881-5a978c9f55a6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-4ab2c namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: 2187e5dc-ba22-4b44-95fd-5156ea40fc7e kind: Event lastTimestamp: "2026-04-16T17:00:49Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-4ab2c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-4ab2c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:00:49Z" name: isvc-sklearn-graph-raw-hpa-4ab2c.18a6e4f6664ba96f namespace: kserve-ci-e2e-test resourceVersion: "15265" uid: 1d5f51a2-91c3-49e5-abcf-15745a43f9c9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-4ab2c namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: 2187e5dc-ba22-4b44-95fd-5156ea40fc7e kind: Event lastTimestamp: "2026-04-16T17:00:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-4ab2c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:00:49Z" name: isvc-sklearn-graph-raw-hpa-4ab2c.18a6e4f6664c0c16 namespace: kserve-ci-e2e-test resourceVersion: "15268" uid: cdb05d50-cbf7-45c5-9c18-1aebedcfb428 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-4ab2c namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: 2187e5dc-ba22-4b44-95fd-5156ea40fc7e kind: Event lastTimestamp: "2026-04-16T17:02:06Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-4ab2c] is Ready metadata: creationTimestamp: "2026-04-16T17:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:02:06Z" name: isvc-sklearn-graph-raw-hpa-4ab2c.18a6e50850ca0fa5 namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: aa2a185d-c8a9-43a6-8a24-47d4607e0111 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:03:50.565448Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c-rqscw namespace: kserve-ci-e2e-test resourceVersion: "16980" uid: 2e9ca9e5-8d6f-44e8-a9d0-147483751715 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c-rqscw to ip-10-0-130-1.ec2.internal metadata: creationTimestamp: "2026-04-16T17:03:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:03:50Z" name: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c-rqscw.18a6e52072693b6d namespace: kserve-ci-e2e-test resourceVersion: "16985" uid: c496c858-0cc0-48e9-a5c9-493ab49d5aa0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f7c99d688-jwz56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c-rqscw namespace: kserve-ci-e2e-test resourceVersion: "16989" uid: 2e9ca9e5-8d6f-44e8-a9d0-147483751715 kind: Event lastTimestamp: "2026-04-16T17:03:50Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:03:50Z" name: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c-rqscw.18a6e5208b4d1f0b namespace: kserve-ci-e2e-test resourceVersion: "17000" uid: 94de8082-12f2-455c-ae80-22b473172f50 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c-rqscw namespace: kserve-ci-e2e-test resourceVersion: "16982" uid: 2e9ca9e5-8d6f-44e8-a9d0-147483751715 kind: Event lastTimestamp: "2026-04-16T17:03:51Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ab565fe3972fd5cd2929b41b0a6a9face626513b818fecb7a38edcdfe0857061" already present on machine metadata: creationTimestamp: "2026-04-16T17:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:03:51Z" name: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c-rqscw.18a6e5208c83b316 namespace: kserve-ci-e2e-test resourceVersion: "17002" uid: de393a1d-2f00-4c84-8b2b-3c384acac8c6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c-rqscw namespace: kserve-ci-e2e-test resourceVersion: "16982" uid: 2e9ca9e5-8d6f-44e8-a9d0-147483751715 kind: Event lastTimestamp: "2026-04-16T17:03:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T17:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:03:51Z" name: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c-rqscw.18a6e5208f3fea60 namespace: kserve-ci-e2e-test resourceVersion: "17004" uid: ed65045b-417d-4327-8896-2401ba725e4d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c-rqscw namespace: kserve-ci-e2e-test resourceVersion: "16982" uid: 2e9ca9e5-8d6f-44e8-a9d0-147483751715 kind: Event lastTimestamp: "2026-04-16T17:03:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T17:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:03:51Z" name: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c-rqscw.18a6e5208faf53da namespace: kserve-ci-e2e-test resourceVersion: "17005" uid: f3da21c6-a06b-4e86-a565-550d30d8f2a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c-rqscw namespace: kserve-ci-e2e-test resourceVersion: "16982" uid: 2e9ca9e5-8d6f-44e8-a9d0-147483751715 kind: Event lastTimestamp: "2026-04-16T17:03:55Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-16T17:03:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:03:55Z" name: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c-rqscw.18a6e52196b6c7b0 namespace: kserve-ci-e2e-test resourceVersion: "17042" uid: f40067cb-18ee-4f63-810f-51609de58e43 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c-rqscw namespace: kserve-ci-e2e-test resourceVersion: "16982" uid: 2e9ca9e5-8d6f-44e8-a9d0-147483751715 kind: Event lastTimestamp: "2026-04-16T17:03:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T17:03:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:03:55Z" name: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c-rqscw.18a6e52199e56cff namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: 211f110b-e833-4b1c-b352-cf17b91d1750 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c-rqscw namespace: kserve-ci-e2e-test resourceVersion: "16982" uid: 2e9ca9e5-8d6f-44e8-a9d0-147483751715 kind: Event lastTimestamp: "2026-04-16T17:03:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T17:03:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:03:55Z" name: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c-rqscw.18a6e5219a597ffe namespace: kserve-ci-e2e-test resourceVersion: "17047" uid: e382a05b-c488-4351-9b73-5f218e7bd4c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T17:03:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c-rqscw namespace: kserve-ci-e2e-test resourceVersion: "16982" uid: 2e9ca9e5-8d6f-44e8-a9d0-147483751715 kind: Event lastTimestamp: "2026-04-16T17:05:46Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T17:03:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:05:46Z" name: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c-rqscw.18a6e521d29afb9f namespace: kserve-ci-e2e-test resourceVersion: "17837" uid: 9f00238b-453a-4962-9408-7bcbd9d0870a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c namespace: kserve-ci-e2e-test resourceVersion: "16967" uid: ec46f57d-3ed7-43d2-aca7-40fd4cfc5ef8 kind: Event lastTimestamp: "2026-04-16T17:03:50Z" message: 'Created pod: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c-rqscw' metadata: creationTimestamp: "2026-04-16T17:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:03:50Z" name: isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c.18a6e52071f881b2 namespace: kserve-ci-e2e-test resourceVersion: "16981" uid: 5f893e00-d31c-4cb6-a16b-551ca63e250c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-ae117-predictor namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: bf037757-38cb-489c-b570-4a72d2dea9cf kind: Event lastTimestamp: "2026-04-16T17:03:50Z" message: Scaled up replica set isvc-sklearn-scale-raw-ae117-predictor-5db48cf96c from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:03:50Z" name: isvc-sklearn-scale-raw-ae117-predictor.18a6e5206ec11980 namespace: kserve-ci-e2e-test resourceVersion: "16969" uid: 6081005a-7463-4c89-b003-e822c2c1ffad reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-ae117-predictor namespace: kserve-ci-e2e-test resourceVersion: "16966" uid: 56573859-b86e-4545-b895-5b7eea2cc65d kind: Event lastTimestamp: "2026-04-16T17:04:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:04:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:04:05Z" name: isvc-sklearn-scale-raw-ae117-predictor.18a6e523ed30133c namespace: kserve-ci-e2e-test resourceVersion: "17118" uid: 109bcca9-3e71-4b3b-aecb-beb35b55456c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-ae117-predictor namespace: kserve-ci-e2e-test resourceVersion: "16966" uid: 56573859-b86e-4545-b895-5b7eea2cc65d kind: Event lastTimestamp: "2026-04-16T17:04:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:04:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:04:05Z" name: isvc-sklearn-scale-raw-ae117-predictor.18a6e523ed8734d8 namespace: kserve-ci-e2e-test resourceVersion: "17119" uid: 2aa8c2a2-8194-436a-a8a6-160d92e76e3c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T17:04:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-ae117-predictor namespace: kserve-ci-e2e-test resourceVersion: "17117" uid: 56573859-b86e-4545-b895-5b7eea2cc65d kind: Event lastTimestamp: "2026-04-16T17:06:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:06:20Z" name: isvc-sklearn-scale-raw-ae117-predictor.18a6e5276c38ff78 namespace: kserve-ci-e2e-test resourceVersion: "18146" uid: bdef3145-04bb-4979-a9df-89071da917d0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T17:04:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-ae117-predictor namespace: kserve-ci-e2e-test resourceVersion: "17117" uid: 56573859-b86e-4545-b895-5b7eea2cc65d kind: Event lastTimestamp: "2026-04-16T17:06:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:06:20Z" name: isvc-sklearn-scale-raw-ae117-predictor.18a6e5276ca6a37e namespace: kserve-ci-e2e-test resourceVersion: "18147" uid: aad8b571-12c2-4ff6-ae73-79e64326706a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T17:03:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-ae117 namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: d9f74a7f-6370-407f-a98f-1dd58db5a2ce kind: Event lastTimestamp: "2026-04-16T17:05:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T17:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:05:56Z" name: isvc-sklearn-scale-raw-ae117.18a6e5206e9946ea namespace: kserve-ci-e2e-test resourceVersion: "17919" uid: 9d4fcdbb-7676-4a99-907c-8bf3224b95dc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:03:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-ae117 namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: d9f74a7f-6370-407f-a98f-1dd58db5a2ce kind: Event lastTimestamp: "2026-04-16T17:03:50Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-ae117": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-ae117": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:03:50Z" name: isvc-sklearn-scale-raw-ae117.18a6e5206f0df1ac namespace: kserve-ci-e2e-test resourceVersion: "16990" uid: 86a4f5d9-3066-4b76-a7b6-cfb95b39f351 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:03:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-ae117 namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: d9f74a7f-6370-407f-a98f-1dd58db5a2ce kind: Event lastTimestamp: "2026-04-16T17:03:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-ae117": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:03:50Z" name: isvc-sklearn-scale-raw-ae117.18a6e5206f0e352c namespace: kserve-ci-e2e-test resourceVersion: "16991" uid: 25079ec2-6bd1-440f-896b-338aba4ca6a5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-ae117 namespace: kserve-ci-e2e-test resourceVersion: "17917" uid: d9f74a7f-6370-407f-a98f-1dd58db5a2ce kind: Event lastTimestamp: "2026-04-16T17:05:56Z" message: InferenceService [isvc-sklearn-scale-raw-ae117] is Ready metadata: creationTimestamp: "2026-04-16T17:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:05:56Z" name: isvc-sklearn-scale-raw-ae117.18a6e53dc99b284f namespace: kserve-ci-e2e-test resourceVersion: "17918" uid: d58ca2b2-aa8e-4496-aadc-e5180a7c7182 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:59:09.807686Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4 namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 471d0f31-b565-488b-bb21-1a41075488e3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4 to ip-10-0-130-1.ec2.internal metadata: creationTimestamp: "2026-04-16T16:59:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:59:09Z" name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4.18a6e4df13f2c4f2 namespace: kserve-ci-e2e-test resourceVersion: "14359" uid: c3efcc22-b726-4a99-aecc-bcf393bd4b46 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f7c99d688-jwz56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4 namespace: kserve-ci-e2e-test resourceVersion: "14363" uid: 471d0f31-b565-488b-bb21-1a41075488e3 kind: Event lastTimestamp: "2026-04-16T16:59:10Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:59:10Z" name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4.18a6e4df2db6fb83 namespace: kserve-ci-e2e-test resourceVersion: "14377" uid: d27ce508-52e4-4859-88a8-756d3c97b8c4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4 namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 471d0f31-b565-488b-bb21-1a41075488e3 kind: Event lastTimestamp: "2026-04-16T16:59:10Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ab565fe3972fd5cd2929b41b0a6a9face626513b818fecb7a38edcdfe0857061" already present on machine metadata: creationTimestamp: "2026-04-16T16:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:59:10Z" name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4.18a6e4df300d6eb4 namespace: kserve-ci-e2e-test resourceVersion: "14382" uid: 8a46f3ca-b6c7-43aa-aaa2-e14a0991da26 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4 namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 471d0f31-b565-488b-bb21-1a41075488e3 kind: Event lastTimestamp: "2026-04-16T16:59:10Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T16:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:59:10Z" name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4.18a6e4df3243bb63 namespace: kserve-ci-e2e-test resourceVersion: "14385" uid: 19cd2a5d-a556-4541-9dbf-2e61ec9bc355 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4 namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 471d0f31-b565-488b-bb21-1a41075488e3 kind: Event lastTimestamp: "2026-04-16T16:59:10Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T16:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:59:10Z" name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4.18a6e4df32ba6045 namespace: kserve-ci-e2e-test resourceVersion: "14386" uid: 99279ff8-44c7-4507-a583-03ddfc55ca4d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4 namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 471d0f31-b565-488b-bb21-1a41075488e3 kind: Event lastTimestamp: "2026-04-16T16:59:14Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-16T16:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:59:14Z" name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4.18a6e4e02bf92a12 namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 73936597-d3ba-410f-b51d-68a9f60e93ac reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4 namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 471d0f31-b565-488b-bb21-1a41075488e3 kind: Event lastTimestamp: "2026-04-16T16:59:30Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 16.214s (16.214s including waiting). Image size: 1306229499 bytes.' metadata: creationTimestamp: "2026-04-16T16:59:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:59:30Z" name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4.18a6e4e3f26df61c namespace: kserve-ci-e2e-test resourceVersion: "14563" uid: 16d69461-9c0a-4672-b939-a2b360607d5a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4 namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 471d0f31-b565-488b-bb21-1a41075488e3 kind: Event lastTimestamp: "2026-04-16T16:59:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:59:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:59:30Z" name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4.18a6e4e3f5b7001c namespace: kserve-ci-e2e-test resourceVersion: "14564" uid: 9b2eb35f-1b1b-4ec5-800d-204f438ca551 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4 namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 471d0f31-b565-488b-bb21-1a41075488e3 kind: Event lastTimestamp: "2026-04-16T16:59:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:59:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:59:30Z" name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4.18a6e4e3f63c8076 namespace: kserve-ci-e2e-test resourceVersion: "14565" uid: b432a571-3208-4927-8000-aef155adc4e7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T16:59:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4 namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 471d0f31-b565-488b-bb21-1a41075488e3 kind: Event lastTimestamp: "2026-04-16T17:00:52Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:52Z" name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4.18a6e4e425402d56 namespace: kserve-ci-e2e-test resourceVersion: "15376" uid: 9acf6e7b-0018-44aa-8d99-ccdb8f7333fb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4 namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 471d0f31-b565-488b-bb21-1a41075488e3 kind: Event lastTimestamp: "2026-04-16T17:00:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T17:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:50Z" name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4.18a6e4f66da58788 namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 1129cc0d-c85b-4a57-aa26-8816aa03adc8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c namespace: kserve-ci-e2e-test resourceVersion: "14334" uid: 43f3dc9a-27b4-4a3e-b640-b3bb99669117 kind: Event lastTimestamp: "2026-04-16T16:59:09Z" message: 'Created pod: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c-r6jf4' metadata: creationTimestamp: "2026-04-16T16:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:59:09Z" name: isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c.18a6e4df137ca8a0 namespace: kserve-ci-e2e-test resourceVersion: "14355" uid: 4e88acce-a376-4fce-a001-08745573f24f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:59:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-231d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14331" uid: 5ce144c5-84e8-4d9f-bc37-27c0e351bbf1 kind: Event lastTimestamp: "2026-04-16T16:59:09Z" message: Scaled up replica set isvc-xgboost-graph-raw-231d1-predictor-5f9bd8476c from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:59:09Z" name: isvc-xgboost-graph-raw-231d1-predictor.18a6e4df10d61085 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: e1b5a566-7a95-49d1-996a-e396cc8b8a2f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T16:59:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-231d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: b894cc19-1898-479a-a0dc-c37785f06456 kind: Event lastTimestamp: "2026-04-16T17:00:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:00:09Z" name: isvc-xgboost-graph-raw-231d1-predictor.18a6e4e28f48decd namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 1ef877b4-b829-44cc-b668-993ed16993b1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T16:59:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-231d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: b894cc19-1898-479a-a0dc-c37785f06456 kind: Event lastTimestamp: "2026-04-16T17:00:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:00:09Z" name: isvc-xgboost-graph-raw-231d1-predictor.18a6e4e28fabc0cf namespace: kserve-ci-e2e-test resourceVersion: "14858" uid: 78459d7a-fe86-4261-885d-41e792611ea6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:00:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-231d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: b894cc19-1898-479a-a0dc-c37785f06456 kind: Event lastTimestamp: "2026-04-16T17:00:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:00:39Z" name: isvc-xgboost-graph-raw-231d1-predictor.18a6e4f08a764ca0 namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: c34b490e-a32c-4a5d-ae64-8d6aa35a63fc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:00:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-231d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: b894cc19-1898-479a-a0dc-c37785f06456 kind: Event lastTimestamp: "2026-04-16T17:00:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:00:39Z" name: isvc-xgboost-graph-raw-231d1-predictor.18a6e4f08adfb6f1 namespace: kserve-ci-e2e-test resourceVersion: "15119" uid: 6d1a5c6e-7f19-484b-b3d6-b6e098733420 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T16:59:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-231d1 namespace: kserve-ci-e2e-test resourceVersion: "14323" uid: 36c8052f-0464-49b6-a8a9-0a84eb5b98e6 kind: Event lastTimestamp: "2026-04-16T17:00:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:00:32Z" name: isvc-xgboost-graph-raw-231d1.18a6e4df10abe631 namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: ff59b7ff-20c4-4566-8589-4e5249713ff7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-231d1 namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: 36c8052f-0464-49b6-a8a9-0a84eb5b98e6 kind: Event lastTimestamp: "2026-04-16T17:00:32Z" message: InferenceService [isvc-xgboost-graph-raw-231d1] is Ready metadata: creationTimestamp: "2026-04-16T17:00:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:00:32Z" name: isvc-xgboost-graph-raw-231d1.18a6e4f25cef1fbc namespace: kserve-ci-e2e-test resourceVersion: "15043" uid: 2b81488f-d79a-4d61-8804-0c02d41fb846 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:00:50.112007Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx namespace: kserve-ci-e2e-test resourceVersion: "15315" uid: 21225193-7f3e-447f-b99c-0926020bbdd9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx to ip-10-0-130-1.ec2.internal metadata: creationTimestamp: "2026-04-16T17:00:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:00:50Z" name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx.18a6e4f66e8c48ec namespace: kserve-ci-e2e-test resourceVersion: "15322" uid: 310b2401-9563-48e2-ac07-6eb00bcc08d3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f7c99d688-jwz56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx namespace: kserve-ci-e2e-test resourceVersion: "15328" uid: 21225193-7f3e-447f-b99c-0926020bbdd9 kind: Event lastTimestamp: "2026-04-16T17:00:50Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:00:50Z" name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx.18a6e4f6887dfa15 namespace: kserve-ci-e2e-test resourceVersion: "15344" uid: 7a4ed7a6-a014-4b50-b67b-13dc86b68eb0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: 21225193-7f3e-447f-b99c-0926020bbdd9 kind: Event lastTimestamp: "2026-04-16T17:00:50Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ab565fe3972fd5cd2929b41b0a6a9face626513b818fecb7a38edcdfe0857061" already present on machine metadata: creationTimestamp: "2026-04-16T17:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:50Z" name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx.18a6e4f6899e1247 namespace: kserve-ci-e2e-test resourceVersion: "15347" uid: 6553e303-9c1c-4c95-88f8-ff5653ed0886 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: 21225193-7f3e-447f-b99c-0926020bbdd9 kind: Event lastTimestamp: "2026-04-16T17:00:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T17:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:50Z" name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx.18a6e4f68c6021c3 namespace: kserve-ci-e2e-test resourceVersion: "15349" uid: 568261f2-3e90-4b3b-889b-29fcd6740f92 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: 21225193-7f3e-447f-b99c-0926020bbdd9 kind: Event lastTimestamp: "2026-04-16T17:00:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T17:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:50Z" name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx.18a6e4f68cd79c27 namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: 74f77e70-dcc7-4a87-9049-f95cbc41b3db reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: 21225193-7f3e-447f-b99c-0926020bbdd9 kind: Event lastTimestamp: "2026-04-16T17:00:54Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-16T17:00:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:54Z" name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx.18a6e4f78905042e namespace: kserve-ci-e2e-test resourceVersion: "15398" uid: 0268de6b-605e-4555-b8b1-83059be99f90 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: 21225193-7f3e-447f-b99c-0926020bbdd9 kind: Event lastTimestamp: "2026-04-16T17:00:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T17:00:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:54Z" name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx.18a6e4f78c9bdc3f namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: 5d17895b-9d44-4e5b-9b2c-8b7a330e41aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: 21225193-7f3e-447f-b99c-0926020bbdd9 kind: Event lastTimestamp: "2026-04-16T17:00:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T17:00:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:54Z" name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx.18a6e4f78d272c32 namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 0df57e81-a229-4e09-a4e8-4c3cf2a721e5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T17:00:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: 21225193-7f3e-447f-b99c-0926020bbdd9 kind: Event lastTimestamp: "2026-04-16T17:01:46Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T17:00:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:01:46Z" name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx.18a6e4f7c5078989 namespace: kserve-ci-e2e-test resourceVersion: "15802" uid: 61da41cf-6f82-4d79-99c0-d7620fd15c74 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: 21225193-7f3e-447f-b99c-0926020bbdd9 kind: Event lastTimestamp: "2026-04-16T17:02:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T17:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:30Z" name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx.18a6e50dc500ea17 namespace: kserve-ci-e2e-test resourceVersion: "16257" uid: fa3572df-ecf1-4cfc-be4e-3a3157900f03 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff namespace: kserve-ci-e2e-test resourceVersion: "15289" uid: a4f85ab5-e261-43f4-bd6f-77f1e34885a2 kind: Event lastTimestamp: "2026-04-16T17:00:50Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff-hcdmx' metadata: creationTimestamp: "2026-04-16T17:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:00:50Z" name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff.18a6e4f66e16bd22 namespace: kserve-ci-e2e-test resourceVersion: "15317" uid: adcd62f0-cc33-452a-a0df-61ab27a10b8b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15285" uid: c5f3541d-515e-41c1-ae8c-9f6895b0fe17 kind: Event lastTimestamp: "2026-04-16T17:00:50Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-4ab2c-predictor-68cc9b64ff from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:00:50Z" name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor.18a6e4f66a9ed0ff namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: 3c83d2a0-3e9d-457f-bd8a-846183045d30 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:01:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 6190521c-9416-4504-9d4f-b3645ad2ae80 kind: Event lastTimestamp: "2026-04-16T17:01:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:01:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:01:05Z" name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor.18a6e4f9e90dd1b8 namespace: kserve-ci-e2e-test resourceVersion: "15491" uid: 36f85bc7-5bd7-45e2-95f3-4d80a3a9dbc4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:01:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 6190521c-9416-4504-9d4f-b3645ad2ae80 kind: Event lastTimestamp: "2026-04-16T17:01:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:01:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:01:05Z" name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor.18a6e4f9e97471c2 namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: 9ebadb0e-13b5-4833-b8dd-407d091efb4c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T17:01:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15492" uid: 6190521c-9416-4504-9d4f-b3645ad2ae80 kind: Event lastTimestamp: "2026-04-16T17:02:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:02:20Z" name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor.18a6e4fd6816da92 namespace: kserve-ci-e2e-test resourceVersion: "16081" uid: 8d5e80bc-cf74-49e2-8baf-488357928605 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T17:01:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15492" uid: 6190521c-9416-4504-9d4f-b3645ad2ae80 kind: Event lastTimestamp: "2026-04-16T17:02:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:02:20Z" name: isvc-xgboost-graph-raw-hpa-4ab2c-predictor.18a6e4fd687a726b namespace: kserve-ci-e2e-test resourceVersion: "16083" uid: f4fbc2f1-1a6e-4d00-9c1b-63b3c7d58699 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T17:00:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-4ab2c namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: 828ef291-bd03-46fa-b993-a0f204012a40 kind: Event lastTimestamp: "2026-04-16T17:01:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T17:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:01:56Z" name: isvc-xgboost-graph-raw-hpa-4ab2c.18a6e4f66a7f47bf namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: 7c9058f0-d2c3-4db3-ab45-3bfd5655cd8e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:01:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-4ab2c namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: 828ef291-bd03-46fa-b993-a0f204012a40 kind: Event lastTimestamp: "2026-04-16T17:01:56Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-4ab2c] is Ready metadata: creationTimestamp: "2026-04-16T17:01:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:01:56Z" name: isvc-xgboost-graph-raw-hpa-4ab2c.18a6e505fd28d913 namespace: kserve-ci-e2e-test resourceVersion: "15881" uid: 90ddba1c-d1d8-4405-9d24-e4223e884d70 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:02:30.302636Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-dc743-predictor-565764948c-52wnv namespace: kserve-ci-e2e-test resourceVersion: "16232" uid: 38ae0f9c-2d58-4bac-a4bc-9e987a6aa202 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-dc743-predictor-565764948c-52wnv to ip-10-0-131-63.ec2.internal metadata: creationTimestamp: "2026-04-16T17:02:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:02:30Z" name: message-dumper-raw-dc743-predictor-565764948c-52wnv.18a6e50dc25ff462 namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 47b4a5c5-24cd-4cc5-8f9a-16535fd976a4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f7c99d688-jwz56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:30Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-dc743-predictor-565764948c-52wnv namespace: kserve-ci-e2e-test resourceVersion: "16247" uid: 38ae0f9c-2d58-4bac-a4bc-9e987a6aa202 kind: Event lastTimestamp: "2026-04-16T17:02:30Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:02:30Z" name: message-dumper-raw-dc743-predictor-565764948c-52wnv.18a6e50dcb5f2cb4 namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: 98ca4bdd-b67e-4143-8692-2ec3ecce57d2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-dc743-predictor-565764948c-52wnv namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 38ae0f9c-2d58-4bac-a4bc-9e987a6aa202 kind: Event lastTimestamp: "2026-04-16T17:02:30Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-16T17:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:30Z" name: message-dumper-raw-dc743-predictor-565764948c-52wnv.18a6e50dcca68ca9 namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: c7a40c6f-2083-495a-8644-ccdb7ad0d34b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-63.ec2.internal source: component: kubelet host: ip-10-0-131-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-dc743-predictor-565764948c-52wnv namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 38ae0f9c-2d58-4bac-a4bc-9e987a6aa202 kind: Event lastTimestamp: "2026-04-16T17:02:31Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.009s (1.009s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-16T17:02:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:31Z" name: message-dumper-raw-dc743-predictor-565764948c-52wnv.18a6e50e08cbdb27 namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 9095715e-a21a-458a-b770-23d0cdfdfe9a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-63.ec2.internal source: component: kubelet host: ip-10-0-131-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-dc743-predictor-565764948c-52wnv namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 38ae0f9c-2d58-4bac-a4bc-9e987a6aa202 kind: Event lastTimestamp: "2026-04-16T17:02:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T17:02:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:31Z" name: message-dumper-raw-dc743-predictor-565764948c-52wnv.18a6e50e101fd8a4 namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: 27976da8-66f5-485f-aacd-a856a7adc393 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-63.ec2.internal source: component: kubelet host: ip-10-0-131-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-dc743-predictor-565764948c-52wnv namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 38ae0f9c-2d58-4bac-a4bc-9e987a6aa202 kind: Event lastTimestamp: "2026-04-16T17:02:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T17:02:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:31Z" name: message-dumper-raw-dc743-predictor-565764948c-52wnv.18a6e50e109586cf namespace: kserve-ci-e2e-test resourceVersion: "16274" uid: eb4b3308-8a62-47d3-917e-2d6cb269e769 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-63.ec2.internal source: component: kubelet host: ip-10-0-131-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-dc743-predictor-565764948c namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: c474e6bb-9cfa-4019-9f02-5e19ba34404e kind: Event lastTimestamp: "2026-04-16T17:02:30Z" message: 'Created pod: message-dumper-raw-dc743-predictor-565764948c-52wnv' metadata: creationTimestamp: "2026-04-16T17:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:02:30Z" name: message-dumper-raw-dc743-predictor-565764948c.18a6e50dc1d9c67c namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 7b5bd149-49b1-4d28-a9d4-6a8ab60d9f13 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-dc743-predictor namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: c32b052a-0d3b-472c-b2a1-f3e201acc90e kind: Event lastTimestamp: "2026-04-16T17:02:30Z" message: Scaled up replica set message-dumper-raw-dc743-predictor-565764948c from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:02:30Z" name: message-dumper-raw-dc743-predictor.18a6e50dbef95496 namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: 3ef870a0-123a-4f23-a6ec-ce601ea1ec2f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-dc743-predictor namespace: kserve-ci-e2e-test resourceVersion: "16210" uid: 395c4343-3894-4ada-b895-a3a54a99c255 kind: Event lastTimestamp: "2026-04-16T17:02:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:02:45Z" name: message-dumper-raw-dc743-predictor.18a6e5113dc26af6 namespace: kserve-ci-e2e-test resourceVersion: "16444" uid: 10f6ceed-45d2-44ce-82cc-a615cf060486 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-dc743-predictor namespace: kserve-ci-e2e-test resourceVersion: "16210" uid: 395c4343-3894-4ada-b895-a3a54a99c255 kind: Event lastTimestamp: "2026-04-16T17:02:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:02:45Z" name: message-dumper-raw-dc743-predictor.18a6e5113e2ff3f3 namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: 58370dfd-fd63-4fd0-823f-84775aa4254f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-dc743-predictor namespace: kserve-ci-e2e-test resourceVersion: "16445" uid: 395c4343-3894-4ada-b895-a3a54a99c255 kind: Event lastTimestamp: "2026-04-16T17:03:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:03:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:03:00Z" name: message-dumper-raw-dc743-predictor.18a6e514bcc98499 namespace: kserve-ci-e2e-test resourceVersion: "16575" uid: 0a1faff3-0cf6-460d-abb6-7c77d1c56b8d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-dc743-predictor namespace: kserve-ci-e2e-test resourceVersion: "16445" uid: 395c4343-3894-4ada-b895-a3a54a99c255 kind: Event lastTimestamp: "2026-04-16T17:03:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:03:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:03:00Z" name: message-dumper-raw-dc743-predictor.18a6e514bd2b1dc4 namespace: kserve-ci-e2e-test resourceVersion: "16577" uid: 17668925-ec79-4636-af2b-fd22965e517b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T17:02:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-dc743 namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: facd8c35-cb20-4008-974e-a7f73555c8ef kind: Event lastTimestamp: "2026-04-16T17:02:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T17:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:02:32Z" name: message-dumper-raw-dc743.18a6e50dbefd3e9a namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: 5141a6cd-aeeb-4ae0-9232-debeecf40b73 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-dc743 namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: facd8c35-cb20-4008-974e-a7f73555c8ef kind: Event lastTimestamp: "2026-04-16T17:02:30Z" message: 'Failed to update status for InferenceService "message-dumper-raw-dc743": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-dc743": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:02:30Z" name: message-dumper-raw-dc743.18a6e50dbfb75625 namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: b8fa1a4c-070b-4a46-bd16-abdb6579dd49 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-dc743 namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: facd8c35-cb20-4008-974e-a7f73555c8ef kind: Event lastTimestamp: "2026-04-16T17:02:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-dc743": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:02:30Z" name: message-dumper-raw-dc743.18a6e50dbfb7a786 namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: cb03e150-2926-4d0c-b2b4-59cafd7f6f34 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-dc743 namespace: kserve-ci-e2e-test resourceVersion: "16291" uid: facd8c35-cb20-4008-974e-a7f73555c8ef kind: Event lastTimestamp: "2026-04-16T17:02:32Z" message: InferenceService [message-dumper-raw-dc743] is Ready metadata: creationTimestamp: "2026-04-16T17:02:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:02:32Z" name: message-dumper-raw-dc743.18a6e50e43d85049 namespace: kserve-ci-e2e-test resourceVersion: "16292" uid: 152e928e-05c8-47eb-bc64-157ce312b9f4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:00:39.786664Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-231d1-5878d8457d-nxjqn namespace: kserve-ci-e2e-test resourceVersion: "15108" uid: 53564292-574b-468b-9b6d-e4b43ee10ca7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-231d1-5878d8457d-nxjqn to ip-10-0-128-130.ec2.internal metadata: creationTimestamp: "2026-04-16T17:00:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:00:39Z" name: model-chainer-raw-231d1-5878d8457d-nxjqn.18a6e4f4071c1132 namespace: kserve-ci-e2e-test resourceVersion: "15113" uid: b9ee909b-47dc-412b-9a54-d5a7c1f997a6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f7c99d688-jwz56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:39Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-231d1-5878d8457d-nxjqn namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: 53564292-574b-468b-9b6d-e4b43ee10ca7 kind: Event lastTimestamp: "2026-04-16T17:00:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-231d1-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T17:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:39Z" name: model-chainer-raw-231d1-5878d8457d-nxjqn.18a6e4f41386472b namespace: kserve-ci-e2e-test resourceVersion: "15127" uid: d71ab394-32b2-479c-9ce6-5746030a63f4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-130.ec2.internal source: component: kubelet host: ip-10-0-128-130.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:40Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-231d1-5878d8457d-nxjqn namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: 53564292-574b-468b-9b6d-e4b43ee10ca7 kind: Event lastTimestamp: "2026-04-16T17:00:40Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:00:40Z" name: model-chainer-raw-231d1-5878d8457d-nxjqn.18a6e4f443c3bd77 namespace: kserve-ci-e2e-test resourceVersion: "15137" uid: 19d0e01e-a367-468e-98a4-77f0694ed322 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-231d1} kind: Pod name: model-chainer-raw-231d1-5878d8457d-nxjqn namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: 53564292-574b-468b-9b6d-e4b43ee10ca7 kind: Event lastTimestamp: "2026-04-16T17:00:40Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:f39a3bd8070dd917cb68f25dddd738d9289c8ff90ef6b585e516f057ca7abea1" metadata: creationTimestamp: "2026-04-16T17:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:40Z" name: model-chainer-raw-231d1-5878d8457d-nxjqn.18a6e4f4452fd502 namespace: kserve-ci-e2e-test resourceVersion: "15139" uid: a6539efc-ebab-4bcc-8b9b-cfbd6476962e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-130.ec2.internal source: component: kubelet host: ip-10-0-128-130.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-231d1} kind: Pod name: model-chainer-raw-231d1-5878d8457d-nxjqn namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: 53564292-574b-468b-9b6d-e4b43ee10ca7 kind: Event lastTimestamp: "2026-04-16T17:00:43Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:f39a3bd8070dd917cb68f25dddd738d9289c8ff90ef6b585e516f057ca7abea1" in 2.262s (2.262s including waiting). Image size: 216231033 bytes.' metadata: creationTimestamp: "2026-04-16T17:00:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:43Z" name: model-chainer-raw-231d1-5878d8457d-nxjqn.18a6e4f4cc03e9a3 namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: cdb298b4-2138-4a54-a5a0-a661e34f540d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-130.ec2.internal source: component: kubelet host: ip-10-0-128-130.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-231d1} kind: Pod name: model-chainer-raw-231d1-5878d8457d-nxjqn namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: 53564292-574b-468b-9b6d-e4b43ee10ca7 kind: Event lastTimestamp: "2026-04-16T17:00:43Z" message: 'Created container: model-chainer-raw-231d1' metadata: creationTimestamp: "2026-04-16T17:00:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:43Z" name: model-chainer-raw-231d1-5878d8457d-nxjqn.18a6e4f4cf09e987 namespace: kserve-ci-e2e-test resourceVersion: "15155" uid: edf679fe-738a-4e27-9179-be1cc8f66602 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-130.ec2.internal source: component: kubelet host: ip-10-0-128-130.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-231d1} kind: Pod name: model-chainer-raw-231d1-5878d8457d-nxjqn namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: 53564292-574b-468b-9b6d-e4b43ee10ca7 kind: Event lastTimestamp: "2026-04-16T17:00:43Z" message: Started container model-chainer-raw-231d1 metadata: creationTimestamp: "2026-04-16T17:00:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:43Z" name: model-chainer-raw-231d1-5878d8457d-nxjqn.18a6e4f4cf7f55b8 namespace: kserve-ci-e2e-test resourceVersion: "15156" uid: 6a65e634-1de4-44ec-bc94-49c0986d76b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-130.ec2.internal source: component: kubelet host: ip-10-0-128-130.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-231d1} kind: Pod name: model-chainer-raw-231d1-5878d8457d-nxjqn namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: 53564292-574b-468b-9b6d-e4b43ee10ca7 kind: Event lastTimestamp: "2026-04-16T17:00:49Z" message: Stopping container model-chainer-raw-231d1 metadata: creationTimestamp: "2026-04-16T17:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:00:49Z" name: model-chainer-raw-231d1-5878d8457d-nxjqn.18a6e4f660ddeade namespace: kserve-ci-e2e-test resourceVersion: "15234" uid: 11a6d944-1da0-4b0b-94b6-9ed097304fa2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-130.ec2.internal source: component: kubelet host: ip-10-0-128-130.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T17:00:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-231d1} kind: Pod name: model-chainer-raw-231d1-5878d8457d-nxjqn namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: 53564292-574b-468b-9b6d-e4b43ee10ca7 kind: Event lastTimestamp: "2026-04-16T17:01:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T17:00:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:01:19Z" name: model-chainer-raw-231d1-5878d8457d-nxjqn.18a6e4f783934c68 namespace: kserve-ci-e2e-test resourceVersion: "15600" uid: 126fd9ea-e387-4fca-9818-3d30202c69d9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-130.ec2.internal source: component: kubelet host: ip-10-0-128-130.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-231d1-5878d8457d namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: 1d9879a2-48ae-470e-ae95-1bd8ef58dd6d kind: Event lastTimestamp: "2026-04-16T17:00:39Z" message: 'Created pod: model-chainer-raw-231d1-5878d8457d-nxjqn' metadata: creationTimestamp: "2026-04-16T17:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:00:39Z" name: model-chainer-raw-231d1-5878d8457d.18a6e4f406b2c4cd namespace: kserve-ci-e2e-test resourceVersion: "15110" uid: f830b5b4-9285-422f-b265-1db31b3c7a40 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-231d1 namespace: kserve-ci-e2e-test resourceVersion: "15101" uid: 96afb1aa-455a-418c-b3df-e00d505499d1 kind: Event lastTimestamp: "2026-04-16T17:00:39Z" message: Scaled up replica set model-chainer-raw-231d1-5878d8457d from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:00:39Z" name: model-chainer-raw-231d1.18a6e4f4059a9286 namespace: kserve-ci-e2e-test resourceVersion: "15106" uid: f1082ee7-f4b8-42a5-85c9-da0c17ea0cbe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:00:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-231d1 namespace: kserve-ci-e2e-test resourceVersion: "15122" uid: 0af23279-1d3d-4f98-beff-106064e8e4bd kind: Event lastTimestamp: "2026-04-16T17:00:39Z" message: InferenceGraph [model-chainer-raw-231d1] is Ready metadata: creationTimestamp: "2026-04-16T17:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:00:39Z" name: model-chainer-raw-231d1.18a6e4f40b0f91cf namespace: kserve-ci-e2e-test resourceVersion: "15124" uid: 03dd2e84-b44a-4445-ad36-693c6e55afb7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:02:20.090179Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-4ab2c-7f5758d7db-jtm22 namespace: kserve-ci-e2e-test resourceVersion: "16070" uid: 573278f1-6e5f-4c4c-b915-080b1194d410 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-4ab2c-7f5758d7db-jtm22 to ip-10-0-128-130.ec2.internal metadata: creationTimestamp: "2026-04-16T17:02:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:02:20Z" name: model-chainer-raw-hpa-4ab2c-7f5758d7db-jtm22.18a6e50b61aa38a4 namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: ad910bee-0a30-43bd-9ffc-efcada791400 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f7c99d688-jwz56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:20Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-4ab2c-7f5758d7db-jtm22 namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: 573278f1-6e5f-4c4c-b915-080b1194d410 kind: Event lastTimestamp: "2026-04-16T17:02:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-4ab2c-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T17:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:20Z" name: model-chainer-raw-hpa-4ab2c-7f5758d7db-jtm22.18a6e50b694c0ab5 namespace: kserve-ci-e2e-test resourceVersion: "16088" uid: 7ac7813a-f88e-43d2-b965-782193c62428 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-130.ec2.internal source: component: kubelet host: ip-10-0-128-130.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:21Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-4ab2c-7f5758d7db-jtm22 namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: 573278f1-6e5f-4c4c-b915-080b1194d410 kind: Event lastTimestamp: "2026-04-16T17:02:21Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:02:21Z" name: model-chainer-raw-hpa-4ab2c-7f5758d7db-jtm22.18a6e50baa46b590 namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: 6fa976ea-a523-4983-a243-3a7942343acb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-4ab2c} kind: Pod name: model-chainer-raw-hpa-4ab2c-7f5758d7db-jtm22 namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: 573278f1-6e5f-4c4c-b915-080b1194d410 kind: Event lastTimestamp: "2026-04-16T17:02:21Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f39a3bd8070dd917cb68f25dddd738d9289c8ff90ef6b585e516f057ca7abea1" already present on machine metadata: creationTimestamp: "2026-04-16T17:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:21Z" name: model-chainer-raw-hpa-4ab2c-7f5758d7db-jtm22.18a6e50bab7726e9 namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: cee65101-7c95-4122-89b0-e131c24be90f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-130.ec2.internal source: component: kubelet host: ip-10-0-128-130.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-4ab2c} kind: Pod name: model-chainer-raw-hpa-4ab2c-7f5758d7db-jtm22 namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: 573278f1-6e5f-4c4c-b915-080b1194d410 kind: Event lastTimestamp: "2026-04-16T17:02:21Z" message: 'Created container: model-chainer-raw-hpa-4ab2c' metadata: creationTimestamp: "2026-04-16T17:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:21Z" name: model-chainer-raw-hpa-4ab2c-7f5758d7db-jtm22.18a6e50bae8d66c2 namespace: kserve-ci-e2e-test resourceVersion: "16099" uid: c16919cb-7402-4acf-95ce-a77025cf6aca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-130.ec2.internal source: component: kubelet host: ip-10-0-128-130.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-4ab2c} kind: Pod name: model-chainer-raw-hpa-4ab2c-7f5758d7db-jtm22 namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: 573278f1-6e5f-4c4c-b915-080b1194d410 kind: Event lastTimestamp: "2026-04-16T17:02:21Z" message: Started container model-chainer-raw-hpa-4ab2c metadata: creationTimestamp: "2026-04-16T17:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:21Z" name: model-chainer-raw-hpa-4ab2c-7f5758d7db-jtm22.18a6e50baee6e9ad namespace: kserve-ci-e2e-test resourceVersion: "16100" uid: 1081405e-95f2-46ee-a78e-954804c9c35f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-130.ec2.internal source: component: kubelet host: ip-10-0-128-130.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-4ab2c} kind: Pod name: model-chainer-raw-hpa-4ab2c-7f5758d7db-jtm22 namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: 573278f1-6e5f-4c4c-b915-080b1194d410 kind: Event lastTimestamp: "2026-04-16T17:02:30Z" message: Stopping container model-chainer-raw-hpa-4ab2c metadata: creationTimestamp: "2026-04-16T17:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:30Z" name: model-chainer-raw-hpa-4ab2c-7f5758d7db-jtm22.18a6e50db8ecc014 namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: 8f43c049-815d-4524-bf95-4cb1cf35b501 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-130.ec2.internal source: component: kubelet host: ip-10-0-128-130.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T17:02:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-4ab2c} kind: Pod name: model-chainer-raw-hpa-4ab2c-7f5758d7db-jtm22 namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: 573278f1-6e5f-4c4c-b915-080b1194d410 kind: Event lastTimestamp: "2026-04-16T17:02:58Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T17:02:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:02:58Z" name: model-chainer-raw-hpa-4ab2c-7f5758d7db-jtm22.18a6e50e66f61a84 namespace: kserve-ci-e2e-test resourceVersion: "16556" uid: 5d9b7482-cb5d-4975-a024-a2bfa6ecb02f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-130.ec2.internal source: component: kubelet host: ip-10-0-128-130.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-4ab2c-7f5758d7db namespace: kserve-ci-e2e-test resourceVersion: "16066" uid: eca5b1e6-8669-4110-809b-cc71f19bcdd1 kind: Event lastTimestamp: "2026-04-16T17:02:20Z" message: 'Created pod: model-chainer-raw-hpa-4ab2c-7f5758d7db-jtm22' metadata: creationTimestamp: "2026-04-16T17:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:02:20Z" name: model-chainer-raw-hpa-4ab2c-7f5758d7db.18a6e50b6111b236 namespace: kserve-ci-e2e-test resourceVersion: "16072" uid: d7be3886-284a-495a-86c8-d912864192c9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-4ab2c namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 0a531536-0907-43ee-9af2-ad2ad84836ef kind: Event lastTimestamp: "2026-04-16T17:02:20Z" message: Scaled up replica set model-chainer-raw-hpa-4ab2c-7f5758d7db from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:02:20Z" name: model-chainer-raw-hpa-4ab2c.18a6e50b602a7f77 namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: ca30aaa3-7392-43e6-9282-15d33580577e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:02:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-4ab2c namespace: kserve-ci-e2e-test resourceVersion: "16082" uid: 970e1556-27ac-4f1d-9bd5-9b1996f6a40c kind: Event lastTimestamp: "2026-04-16T17:02:20Z" message: InferenceGraph [model-chainer-raw-hpa-4ab2c] is Ready metadata: creationTimestamp: "2026-04-16T17:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:02:20Z" name: model-chainer-raw-hpa-4ab2c.18a6e50b6443ab81 namespace: kserve-ci-e2e-test resourceVersion: "16085" uid: 6a1da2ff-89d1-41c0-b708-d2761e957cc2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:07:44.283021Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-19113-predictor-6dbb9fd749-wfs4l namespace: kserve-ci-e2e-test resourceVersion: "18972" uid: 92d315dc-af6b-4a1d-8105-bd1156211f7d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-19113-predictor-6dbb9fd749-wfs4l to ip-10-0-130-1.ec2.internal metadata: creationTimestamp: "2026-04-16T17:07:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:07:44Z" name: raw-sklearn-19113-predictor-6dbb9fd749-wfs4l.18a6e556dd10722f namespace: kserve-ci-e2e-test resourceVersion: "18977" uid: 202ad433-3e66-4d79-acad-d4a8a403e3d9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f7c99d688-jwz56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:44Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-19113-predictor-6dbb9fd749-wfs4l namespace: kserve-ci-e2e-test resourceVersion: "18981" uid: 92d315dc-af6b-4a1d-8105-bd1156211f7d kind: Event lastTimestamp: "2026-04-16T17:07:44Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:07:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:07:44Z" name: raw-sklearn-19113-predictor-6dbb9fd749-wfs4l.18a6e556f68b017a namespace: kserve-ci-e2e-test resourceVersion: "18990" uid: 3d15de9f-29a7-46a5-a1b6-f01aa51e81a9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-19113-predictor-6dbb9fd749-wfs4l namespace: kserve-ci-e2e-test resourceVersion: "18975" uid: 92d315dc-af6b-4a1d-8105-bd1156211f7d kind: Event lastTimestamp: "2026-04-16T17:07:44Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ab565fe3972fd5cd2929b41b0a6a9face626513b818fecb7a38edcdfe0857061" already present on machine metadata: creationTimestamp: "2026-04-16T17:07:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:07:44Z" name: raw-sklearn-19113-predictor-6dbb9fd749-wfs4l.18a6e556f7bdf050 namespace: kserve-ci-e2e-test resourceVersion: "18992" uid: 1c02559c-67b8-4c97-8bb7-af95ff2699cd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-19113-predictor-6dbb9fd749-wfs4l namespace: kserve-ci-e2e-test resourceVersion: "18975" uid: 92d315dc-af6b-4a1d-8105-bd1156211f7d kind: Event lastTimestamp: "2026-04-16T17:07:44Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T17:07:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:07:44Z" name: raw-sklearn-19113-predictor-6dbb9fd749-wfs4l.18a6e556faa8432f namespace: kserve-ci-e2e-test resourceVersion: "18994" uid: 43570915-9b54-48d1-b982-abe548e94500 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-19113-predictor-6dbb9fd749-wfs4l namespace: kserve-ci-e2e-test resourceVersion: "18975" uid: 92d315dc-af6b-4a1d-8105-bd1156211f7d kind: Event lastTimestamp: "2026-04-16T17:07:44Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T17:07:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:07:44Z" name: raw-sklearn-19113-predictor-6dbb9fd749-wfs4l.18a6e556fb207c26 namespace: kserve-ci-e2e-test resourceVersion: "18995" uid: 46e9723b-5782-4628-a0e6-2aeef95862f5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-19113-predictor-6dbb9fd749-wfs4l namespace: kserve-ci-e2e-test resourceVersion: "18975" uid: 92d315dc-af6b-4a1d-8105-bd1156211f7d kind: Event lastTimestamp: "2026-04-16T17:07:49Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-16T17:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:07:49Z" name: raw-sklearn-19113-predictor-6dbb9fd749-wfs4l.18a6e55805ea46dc namespace: kserve-ci-e2e-test resourceVersion: "19031" uid: ba09341e-e405-478e-964c-842b100b140f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-19113-predictor-6dbb9fd749-wfs4l namespace: kserve-ci-e2e-test resourceVersion: "18975" uid: 92d315dc-af6b-4a1d-8105-bd1156211f7d kind: Event lastTimestamp: "2026-04-16T17:07:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T17:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:07:49Z" name: raw-sklearn-19113-predictor-6dbb9fd749-wfs4l.18a6e5580e388144 namespace: kserve-ci-e2e-test resourceVersion: "19037" uid: c5628ccc-38a8-4782-8538-8a5daf62dae5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-19113-predictor-6dbb9fd749-wfs4l namespace: kserve-ci-e2e-test resourceVersion: "18975" uid: 92d315dc-af6b-4a1d-8105-bd1156211f7d kind: Event lastTimestamp: "2026-04-16T17:07:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T17:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:07:49Z" name: raw-sklearn-19113-predictor-6dbb9fd749-wfs4l.18a6e5580ed6a3ec namespace: kserve-ci-e2e-test resourceVersion: "19038" uid: 1f91635b-9d37-42bf-a832-f79cc299c3f5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T17:07:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-19113-predictor-6dbb9fd749-wfs4l namespace: kserve-ci-e2e-test resourceVersion: "18975" uid: 92d315dc-af6b-4a1d-8105-bd1156211f7d kind: Event lastTimestamp: "2026-04-16T17:08:51Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T17:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:08:51Z" name: raw-sklearn-19113-predictor-6dbb9fd749-wfs4l.18a6e55841dbd2dc namespace: kserve-ci-e2e-test resourceVersion: "19477" uid: d331f731-c725-4d4d-9f9d-842e8d26794a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-19113-predictor-6dbb9fd749 namespace: kserve-ci-e2e-test resourceVersion: "18956" uid: 38286c85-a6e9-492e-bfae-33dbd514f33f kind: Event lastTimestamp: "2026-04-16T17:07:44Z" message: 'Created pod: raw-sklearn-19113-predictor-6dbb9fd749-wfs4l' metadata: creationTimestamp: "2026-04-16T17:07:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:07:44Z" name: raw-sklearn-19113-predictor-6dbb9fd749.18a6e556dcb10ecf namespace: kserve-ci-e2e-test resourceVersion: "18974" uid: 57557e07-ef36-4548-9483-68cd83a7b0dd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-19113-predictor namespace: kserve-ci-e2e-test resourceVersion: "18954" uid: 12028d52-edc6-4394-b6a5-3a39bd3cb221 kind: Event lastTimestamp: "2026-04-16T17:07:44Z" message: Scaled up replica set raw-sklearn-19113-predictor-6dbb9fd749 from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:07:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:07:44Z" name: raw-sklearn-19113-predictor.18a6e556d9c6e531 namespace: kserve-ci-e2e-test resourceVersion: "18959" uid: abbab0e2-7cf5-4aa7-969c-97f09a28acfa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-19113-predictor namespace: kserve-ci-e2e-test resourceVersion: "18955" uid: d589fa8b-0dd4-46ac-ab50-a34b3e02d469 kind: Event lastTimestamp: "2026-04-16T17:07:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:07:59Z" name: raw-sklearn-19113-predictor.18a6e55a58396cb4 namespace: kserve-ci-e2e-test resourceVersion: "19109" uid: 481b3345-3c0f-48c3-9456-369f2fea7d09 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-19113-predictor namespace: kserve-ci-e2e-test resourceVersion: "18955" uid: d589fa8b-0dd4-46ac-ab50-a34b3e02d469 kind: Event lastTimestamp: "2026-04-16T17:07:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:07:59Z" name: raw-sklearn-19113-predictor.18a6e55a589a8bf5 namespace: kserve-ci-e2e-test resourceVersion: "19111" uid: 41faeee8-e581-43e6-9301-61afc7206408 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T17:08:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-19113-predictor namespace: kserve-ci-e2e-test resourceVersion: "19110" uid: d589fa8b-0dd4-46ac-ab50-a34b3e02d469 kind: Event lastTimestamp: "2026-04-16T17:09:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:08:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:09:29Z" name: raw-sklearn-19113-predictor.18a6e55dd775afcc namespace: kserve-ci-e2e-test resourceVersion: "19819" uid: a00f58e2-9c6a-436b-b95a-eb9c08df477c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T17:08:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-19113-predictor namespace: kserve-ci-e2e-test resourceVersion: "19110" uid: d589fa8b-0dd4-46ac-ab50-a34b3e02d469 kind: Event lastTimestamp: "2026-04-16T17:09:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:08:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:09:29Z" name: raw-sklearn-19113-predictor.18a6e55dd7de270f namespace: kserve-ci-e2e-test resourceVersion: "19820" uid: 90e7ce53-a683-45a6-b569-e0597037c2ea reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T17:07:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-19113 namespace: kserve-ci-e2e-test resourceVersion: "18947" uid: ae29fa18-a5b3-40c5-b54d-67fad170f8d0 kind: Event lastTimestamp: "2026-04-16T17:09:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T17:07:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:09:01Z" name: raw-sklearn-19113.18a6e556d9a27911 namespace: kserve-ci-e2e-test resourceVersion: "19560" uid: 4e54d69b-5de5-4067-9a3a-1a6da48f90f3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-19113 namespace: kserve-ci-e2e-test resourceVersion: "18947" uid: ae29fa18-a5b3-40c5-b54d-67fad170f8d0 kind: Event lastTimestamp: "2026-04-16T17:07:44Z" message: 'Failed to update status for InferenceService "raw-sklearn-19113": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-19113": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:07:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:07:44Z" name: raw-sklearn-19113.18a6e556d9f99c94 namespace: kserve-ci-e2e-test resourceVersion: "18961" uid: 8302ed04-241f-497d-9274-0215634047fa reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:07:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-19113 namespace: kserve-ci-e2e-test resourceVersion: "18947" uid: ae29fa18-a5b3-40c5-b54d-67fad170f8d0 kind: Event lastTimestamp: "2026-04-16T17:07:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-19113": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:07:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:07:44Z" name: raw-sklearn-19113.18a6e556d9f9ee40 namespace: kserve-ci-e2e-test resourceVersion: "18962" uid: ec10d82d-eca9-4a61-82e4-f953330ef4cb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:09:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-19113 namespace: kserve-ci-e2e-test resourceVersion: "19558" uid: ae29fa18-a5b3-40c5-b54d-67fad170f8d0 kind: Event lastTimestamp: "2026-04-16T17:09:01Z" message: InferenceService [raw-sklearn-19113] is Ready metadata: creationTimestamp: "2026-04-16T17:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:09:01Z" name: raw-sklearn-19113.18a6e568ce8b94cf namespace: kserve-ci-e2e-test resourceVersion: "19559" uid: f1f94479-04f8-428d-89b9-9239145531c5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:09:04.469672Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-aa36c-predictor-5b67688557-qw2cs namespace: kserve-ci-e2e-test resourceVersion: "19606" uid: e32be801-b24c-4954-8d62-24bff5604915 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-aa36c-predictor-5b67688557-qw2cs to ip-10-0-130-1.ec2.internal metadata: creationTimestamp: "2026-04-16T17:09:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:09:04Z" name: raw-sklearn-runtime-aa36c-predictor-5b67688557-qw2cs.18a6e569888fa09b namespace: kserve-ci-e2e-test resourceVersion: "19610" uid: 8a013978-8554-409c-9739-200144862cf3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f7c99d688-jwz56 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:09:04Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-aa36c-predictor-5b67688557-qw2cs namespace: kserve-ci-e2e-test resourceVersion: "19614" uid: e32be801-b24c-4954-8d62-24bff5604915 kind: Event lastTimestamp: "2026-04-16T17:09:04Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:09:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:09:04Z" name: raw-sklearn-runtime-aa36c-predictor-5b67688557-qw2cs.18a6e569a1b61299 namespace: kserve-ci-e2e-test resourceVersion: "19623" uid: c5ecefd0-735f-458e-af4a-6e1a622ba926 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:09:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-aa36c-predictor-5b67688557-qw2cs namespace: kserve-ci-e2e-test resourceVersion: "19608" uid: e32be801-b24c-4954-8d62-24bff5604915 kind: Event lastTimestamp: "2026-04-16T17:09:04Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ab565fe3972fd5cd2929b41b0a6a9face626513b818fecb7a38edcdfe0857061" already present on machine metadata: creationTimestamp: "2026-04-16T17:09:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:09:04Z" name: raw-sklearn-runtime-aa36c-predictor-5b67688557-qw2cs.18a6e569a2f9bdae namespace: kserve-ci-e2e-test resourceVersion: "19625" uid: fb7501d1-8796-4cc0-ba86-f50cb662e8e3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:09:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-aa36c-predictor-5b67688557-qw2cs namespace: kserve-ci-e2e-test resourceVersion: "19608" uid: e32be801-b24c-4954-8d62-24bff5604915 kind: Event lastTimestamp: "2026-04-16T17:09:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T17:09:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:09:04Z" name: raw-sklearn-runtime-aa36c-predictor-5b67688557-qw2cs.18a6e569a618e597 namespace: kserve-ci-e2e-test resourceVersion: "19628" uid: bdff118a-bd5c-46ad-9c73-290753c432bf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:09:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-aa36c-predictor-5b67688557-qw2cs namespace: kserve-ci-e2e-test resourceVersion: "19608" uid: e32be801-b24c-4954-8d62-24bff5604915 kind: Event lastTimestamp: "2026-04-16T17:09:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T17:09:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:09:04Z" name: raw-sklearn-runtime-aa36c-predictor-5b67688557-qw2cs.18a6e569a68e7a80 namespace: kserve-ci-e2e-test resourceVersion: "19629" uid: 5685a14f-4f17-4d82-8f5f-87103be5d310 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-aa36c-predictor-5b67688557-qw2cs namespace: kserve-ci-e2e-test resourceVersion: "19608" uid: e32be801-b24c-4954-8d62-24bff5604915 kind: Event lastTimestamp: "2026-04-16T17:09:09Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-16T17:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:09:09Z" name: raw-sklearn-runtime-aa36c-predictor-5b67688557-qw2cs.18a6e56ab735d294 namespace: kserve-ci-e2e-test resourceVersion: "19667" uid: 404bdfc3-d093-427a-8e46-63bc4fc02b85 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-aa36c-predictor-5b67688557-qw2cs namespace: kserve-ci-e2e-test resourceVersion: "19608" uid: e32be801-b24c-4954-8d62-24bff5604915 kind: Event lastTimestamp: "2026-04-16T17:09:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T17:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:09:09Z" name: raw-sklearn-runtime-aa36c-predictor-5b67688557-qw2cs.18a6e56aba9a92de namespace: kserve-ci-e2e-test resourceVersion: "19671" uid: debd09c0-3a8f-4776-bb3d-9764afa34445 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-aa36c-predictor-5b67688557-qw2cs namespace: kserve-ci-e2e-test resourceVersion: "19608" uid: e32be801-b24c-4954-8d62-24bff5604915 kind: Event lastTimestamp: "2026-04-16T17:09:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T17:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:09:09Z" name: raw-sklearn-runtime-aa36c-predictor-5b67688557-qw2cs.18a6e56abb01cb67 namespace: kserve-ci-e2e-test resourceVersion: "19672" uid: 590c6b40-d050-4b59-8038-6c493e052648 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T17:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-aa36c-predictor-5b67688557-qw2cs namespace: kserve-ci-e2e-test resourceVersion: "19608" uid: e32be801-b24c-4954-8d62-24bff5604915 kind: Event lastTimestamp: "2026-04-16T17:10:11Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T17:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:10:11Z" name: raw-sklearn-runtime-aa36c-predictor-5b67688557-qw2cs.18a6e56af322b375 namespace: kserve-ci-e2e-test resourceVersion: "20115" uid: 3d71b9b8-8c49-434c-8f7f-19dbf9e11692 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-1.ec2.internal source: component: kubelet host: ip-10-0-130-1.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:09:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-aa36c-predictor-5b67688557 namespace: kserve-ci-e2e-test resourceVersion: "19594" uid: bb64393c-40da-4cfb-9bc4-1b52814846aa kind: Event lastTimestamp: "2026-04-16T17:09:04Z" message: 'Created pod: raw-sklearn-runtime-aa36c-predictor-5b67688557-qw2cs' metadata: creationTimestamp: "2026-04-16T17:09:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:09:04Z" name: raw-sklearn-runtime-aa36c-predictor-5b67688557.18a6e56988221fbe namespace: kserve-ci-e2e-test resourceVersion: "19607" uid: e2d4b4ec-9d7a-4208-b261-36585178c2a0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:09:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-aa36c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19591" uid: f87e16b8-9030-4b32-8306-adfb6865378c kind: Event lastTimestamp: "2026-04-16T17:09:04Z" message: Scaled up replica set raw-sklearn-runtime-aa36c-predictor-5b67688557 from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:09:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:09:04Z" name: raw-sklearn-runtime-aa36c-predictor.18a6e569855018c2 namespace: kserve-ci-e2e-test resourceVersion: "19596" uid: 540b4168-2f8b-4136-b3a8-9eba73543168 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:09:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-aa36c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19593" uid: ec7351ba-6325-442b-99f1-2b46baa342c0 kind: Event lastTimestamp: "2026-04-16T17:09:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:09:34Z" name: raw-sklearn-runtime-aa36c-predictor.18a6e56d03d782dd namespace: kserve-ci-e2e-test resourceVersion: "19854" uid: 7c06b8bf-03fe-404a-afc5-9d745a7b40fb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:09:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-aa36c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19593" uid: ec7351ba-6325-442b-99f1-2b46baa342c0 kind: Event lastTimestamp: "2026-04-16T17:09:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:09:34Z" name: raw-sklearn-runtime-aa36c-predictor.18a6e56d043a3f68 namespace: kserve-ci-e2e-test resourceVersion: "19855" uid: 89f21d00-f58b-4b56-8f62-c0672969174f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T17:09:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-aa36c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19745" uid: ec7351ba-6325-442b-99f1-2b46baa342c0 kind: Event lastTimestamp: "2026-04-16T17:10:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:09:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:10:49Z" name: raw-sklearn-runtime-aa36c-predictor.18a6e57401d68a45 namespace: kserve-ci-e2e-test resourceVersion: "20430" uid: df1f7629-1bf5-4019-aa11-50ffa031bb5d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T17:09:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-aa36c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19745" uid: ec7351ba-6325-442b-99f1-2b46baa342c0 kind: Event lastTimestamp: "2026-04-16T17:10:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:09:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:10:49Z" name: raw-sklearn-runtime-aa36c-predictor.18a6e5740237cd89 namespace: kserve-ci-e2e-test resourceVersion: "20431" uid: 013c9ef8-7181-441e-8ded-6ba978bb615f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T17:09:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-aa36c namespace: kserve-ci-e2e-test resourceVersion: "19583" uid: 4b5e1cc8-804a-4928-a68e-02176bb54aad kind: Event lastTimestamp: "2026-04-16T17:10:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T17:09:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:10:21Z" name: raw-sklearn-runtime-aa36c.18a6e569852ca2d3 namespace: kserve-ci-e2e-test resourceVersion: "20199" uid: 426a35c1-25c1-43c6-8a93-ab7ad3435509 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:09:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-aa36c namespace: kserve-ci-e2e-test resourceVersion: "19583" uid: 4b5e1cc8-804a-4928-a68e-02176bb54aad kind: Event lastTimestamp: "2026-04-16T17:09:04Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-aa36c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-aa36c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:09:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:09:04Z" name: raw-sklearn-runtime-aa36c.18a6e56985c67b0e namespace: kserve-ci-e2e-test resourceVersion: "19599" uid: 4100d18e-423f-43a5-86c6-c36807383e76 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:09:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-aa36c namespace: kserve-ci-e2e-test resourceVersion: "19583" uid: 4b5e1cc8-804a-4928-a68e-02176bb54aad kind: Event lastTimestamp: "2026-04-16T17:09:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-aa36c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:09:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:09:04Z" name: raw-sklearn-runtime-aa36c.18a6e56985c6ceff namespace: kserve-ci-e2e-test resourceVersion: "19602" uid: 5fdc4c28-eda0-4573-bea2-5bebd41e6fe5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:10:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-aa36c namespace: kserve-ci-e2e-test resourceVersion: "20197" uid: 4b5e1cc8-804a-4928-a68e-02176bb54aad kind: Event lastTimestamp: "2026-04-16T17:10:21Z" message: InferenceService [raw-sklearn-runtime-aa36c] is Ready metadata: creationTimestamp: "2026-04-16T17:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:10:21Z" name: raw-sklearn-runtime-aa36c.18a6e57b7f446c14 namespace: kserve-ci-e2e-test resourceVersion: "20198" uid: 2e2c26f4-0083-4408-a2b6-982c5d56ec28 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "20460"