--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:34:44.676857Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-e681b7-predictor-57cb65b9d8-nljmf namespace: kserve-ci-e2e-test resourceVersion: "18748" uid: 97749758-c530-471e-a18d-cb99b3d0801e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-e681b7-predictor-57cb65b9d8-nljmf to ip-10-0-132-118.ec2.internal metadata: creationTimestamp: "2026-04-24T21:34:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:34:44Z" name: isvc-init-fail-e681b7-predictor-57cb65b9d8-nljmf.18a9688d716aab82 namespace: kserve-ci-e2e-test resourceVersion: "18752" uid: 1171d4c5-b1c1-40f3-9a7d-22d878992b8c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6b5fbb9956-pzrgr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:44Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-e681b7-predictor-57cb65b9d8-nljmf namespace: kserve-ci-e2e-test resourceVersion: "18750" uid: 97749758-c530-471e-a18d-cb99b3d0801e kind: Event lastTimestamp: "2026-04-24T21:34:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-init-fail-e681b7-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:34:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:44Z" name: isvc-init-fail-e681b7-predictor-57cb65b9d8-nljmf.18a9688d7c7d0a58 namespace: kserve-ci-e2e-test resourceVersion: "18763" uid: 3afdf604-c1ac-47fe-bd04-bcfc358d7e76 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-e681b7-predictor-57cb65b9d8-nljmf namespace: kserve-ci-e2e-test resourceVersion: "18756" uid: 97749758-c530-471e-a18d-cb99b3d0801e kind: Event lastTimestamp: "2026-04-24T21:34:45Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:34:45Z" name: isvc-init-fail-e681b7-predictor-57cb65b9d8-nljmf.18a9688dae85b617 namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: b0139a51-f49f-4434-b381-d4e7c4eb9428 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:34:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-e681b7-predictor-57cb65b9d8-nljmf namespace: kserve-ci-e2e-test resourceVersion: "18750" uid: 97749758-c530-471e-a18d-cb99b3d0801e kind: Event lastTimestamp: "2026-04-24T21:34:50Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:85f85c16414082de64de782f602228652c10c5bace08b518e803a7e911c62e5e" already present on machine metadata: creationTimestamp: "2026-04-24T21:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:50Z" name: isvc-init-fail-e681b7-predictor-57cb65b9d8-nljmf.18a9688db001a7e3 namespace: kserve-ci-e2e-test resourceVersion: "18824" uid: 2676d28d-cbce-439e-9f1d-252f9ff9ee0b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:34:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-e681b7-predictor-57cb65b9d8-nljmf namespace: kserve-ci-e2e-test resourceVersion: "18750" uid: 97749758-c530-471e-a18d-cb99b3d0801e kind: Event lastTimestamp: "2026-04-24T21:34:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:50Z" name: isvc-init-fail-e681b7-predictor-57cb65b9d8-nljmf.18a9688db335de35 namespace: kserve-ci-e2e-test resourceVersion: "18826" uid: 69b5086a-59d1-46c1-9737-3c101c67e34e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:34:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-e681b7-predictor-57cb65b9d8-nljmf namespace: kserve-ci-e2e-test resourceVersion: "18750" uid: 97749758-c530-471e-a18d-cb99b3d0801e kind: Event lastTimestamp: "2026-04-24T21:34:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:50Z" name: isvc-init-fail-e681b7-predictor-57cb65b9d8-nljmf.18a9688db3a36a46 namespace: kserve-ci-e2e-test resourceVersion: "18829" uid: 30cc7bab-2159-4d65-853a-f5b0a728415e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-e681b7-predictor-57cb65b9d8-nljmf namespace: kserve-ci-e2e-test resourceVersion: "18750" uid: 97749758-c530-471e-a18d-cb99b3d0801e kind: Event lastTimestamp: "2026-04-24T21:34:54Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-04-24T21:34:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:54Z" name: isvc-init-fail-e681b7-predictor-57cb65b9d8-nljmf.18a9688fc4c63c6c namespace: kserve-ci-e2e-test resourceVersion: "18877" uid: b7d33937-41ad-4b26-adeb-42c100df3c7d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-e681b7-predictor-57cb65b9d8 namespace: kserve-ci-e2e-test resourceVersion: "18739" uid: 8a3347f3-e8f1-46cb-a227-b7cf01bc743e kind: Event lastTimestamp: "2026-04-24T21:34:44Z" message: 'Created pod: isvc-init-fail-e681b7-predictor-57cb65b9d8-nljmf' metadata: creationTimestamp: "2026-04-24T21:34:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:44Z" name: isvc-init-fail-e681b7-predictor-57cb65b9d8.18a9688d70f92f80 namespace: kserve-ci-e2e-test resourceVersion: "18749" uid: 5c4b8eb2-9e71-41e9-bcb9-0c34b3b03645 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-e681b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "18737" uid: f078ac38-4f56-4d8c-afb9-b9c3c1fb9486 kind: Event lastTimestamp: "2026-04-24T21:34:44Z" message: Scaled up replica set isvc-init-fail-e681b7-predictor-57cb65b9d8 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:34:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:44Z" name: isvc-init-fail-e681b7-predictor.18a9688d6e007d25 namespace: kserve-ci-e2e-test resourceVersion: "18741" uid: 03392097-cf80-4f01-8903-84feb3ff8c33 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T21:34:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-e681b7 namespace: kserve-ci-e2e-test resourceVersion: "18709" uid: 87bffca0-ad1f-4794-a24b-10021bb0f890 kind: Event lastTimestamp: "2026-04-24T21:34:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:34:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:34:50Z" name: isvc-init-fail-e681b7.18a9688d6dd7b279 namespace: kserve-ci-e2e-test resourceVersion: "18830" uid: 9f075cfd-74bb-4a38-87e7-60ce0984c9b3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-e681b7 namespace: kserve-ci-e2e-test resourceVersion: "18709" uid: 87bffca0-ad1f-4794-a24b-10021bb0f890 kind: Event lastTimestamp: "2026-04-24T21:34:44Z" message: 'Failed to update status for InferenceService "isvc-init-fail-e681b7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-e681b7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:34:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:34:44Z" name: isvc-init-fail-e681b7.18a9688d6e47e26a namespace: kserve-ci-e2e-test resourceVersion: "18743" uid: 4db27cd3-0dbd-4344-8731-ef7b04e01ecf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-e681b7 namespace: kserve-ci-e2e-test resourceVersion: "18709" uid: 87bffca0-ad1f-4794-a24b-10021bb0f890 kind: Event lastTimestamp: "2026-04-24T21:34:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-e681b7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:34:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:34:44Z" name: isvc-init-fail-e681b7.18a9688d6e48310a namespace: kserve-ci-e2e-test resourceVersion: "18744" uid: a50a20d6-119f-4d27-9b82-d4f4de2dce6a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:29:10.825779Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz namespace: kserve-ci-e2e-test resourceVersion: "15980" uid: 34f17ebf-3db3-42bf-af14-e93e366ee8be kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz to ip-10-0-132-118.ec2.internal metadata: creationTimestamp: "2026-04-24T21:29:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:29:10Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz.18a9683fb65786b9 namespace: kserve-ci-e2e-test resourceVersion: "15986" uid: e4ed3347-4d1a-41f7-8a21-851f1f0e11c9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6b5fbb9956-pzrgr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz namespace: kserve-ci-e2e-test resourceVersion: "15991" uid: 34f17ebf-3db3-42bf-af14-e93e366ee8be kind: Event lastTimestamp: "2026-04-24T21:29:11Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:29:11Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz.18a9683fcf45dc7a namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: a47a05df-6ff6-432f-bf28-4094f8700ff1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 34f17ebf-3db3-42bf-af14-e93e366ee8be kind: Event lastTimestamp: "2026-04-24T21:29:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:85f85c16414082de64de782f602228652c10c5bace08b518e803a7e911c62e5e" already present on machine metadata: creationTimestamp: "2026-04-24T21:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:11Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz.18a9683fd0bfb1f6 namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: e1878dcd-c70f-431c-94cc-66f2552dc4e1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 34f17ebf-3db3-42bf-af14-e93e366ee8be kind: Event lastTimestamp: "2026-04-24T21:29:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:11Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz.18a9683fd3a17edc namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 81dcc915-7021-4942-bdf0-03c4f148cb4e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 34f17ebf-3db3-42bf-af14-e93e366ee8be kind: Event lastTimestamp: "2026-04-24T21:29:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:11Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz.18a9683fd412e4df namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 7a017951-9088-430a-b344-ed7313abc755 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 34f17ebf-3db3-42bf-af14-e93e366ee8be kind: Event lastTimestamp: "2026-04-24T21:29:15Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:15Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz.18a96840e1596034 namespace: kserve-ci-e2e-test resourceVersion: "16040" uid: 6fe3a2aa-3518-45ca-93b8-d03ecf6ed045 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 34f17ebf-3db3-42bf-af14-e93e366ee8be kind: Event lastTimestamp: "2026-04-24T21:29:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:15Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz.18a96840e4132a20 namespace: kserve-ci-e2e-test resourceVersion: "16042" uid: 55840755-961d-4db2-892f-da97d5eb649a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 34f17ebf-3db3-42bf-af14-e93e366ee8be kind: Event lastTimestamp: "2026-04-24T21:29:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:15Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz.18a96840e481549d namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 9d8836c1-13c0-4f41-a98f-65c155e5cedd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 34f17ebf-3db3-42bf-af14-e93e366ee8be kind: Event lastTimestamp: "2026-04-24T21:29:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:15Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz.18a96840e48cf1c3 namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: 7c4969c6-2fd4-466f-8c1f-d2e417c4fcc5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 34f17ebf-3db3-42bf-af14-e93e366ee8be kind: Event lastTimestamp: "2026-04-24T21:29:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:15Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz.18a96840e7a3a92f namespace: kserve-ci-e2e-test resourceVersion: "16050" uid: 6c93da64-690e-4bfb-a64d-3b31dc4e5294 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 34f17ebf-3db3-42bf-af14-e93e366ee8be kind: Event lastTimestamp: "2026-04-24T21:29:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:15Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz.18a96840e811a976 namespace: kserve-ci-e2e-test resourceVersion: "16051" uid: a30ae95a-239c-4c09-ad93-5ef6a6b444f9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 34f17ebf-3db3-42bf-af14-e93e366ee8be kind: Event lastTimestamp: "2026-04-24T21:29:15Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:e6428204baa077dfe80627f4a0f194329877d007e326693ea30a582520538a96" already present on machine metadata: creationTimestamp: "2026-04-24T21:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:15Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz.18a96840e81db63d namespace: kserve-ci-e2e-test resourceVersion: "16052" uid: 9d74b783-3ebd-44c5-a26b-bcd8bde8cad1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 34f17ebf-3db3-42bf-af14-e93e366ee8be kind: Event lastTimestamp: "2026-04-24T21:29:16Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-24T21:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:16Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz.18a96840eb488472 namespace: kserve-ci-e2e-test resourceVersion: "16053" uid: d3acf8e8-6443-4bff-abe7-8ec96a58c220 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 34f17ebf-3db3-42bf-af14-e93e366ee8be kind: Event lastTimestamp: "2026-04-24T21:29:16Z" message: Started container agent metadata: creationTimestamp: "2026-04-24T21:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:16Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz.18a96840ebb2e4f3 namespace: kserve-ci-e2e-test resourceVersion: "16054" uid: 61614fb3-d3ef-4593-9f54-887c2fd6fbd8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-24T21:29:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 34f17ebf-3db3-42bf-af14-e93e366ee8be kind: Event lastTimestamp: "2026-04-24T21:30:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:53Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz.18a968411d500e1c namespace: kserve-ci-e2e-test resourceVersion: "16882" uid: 396f664c-09d5-4e3e-b0c9-1a13653a2a12 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T21:29:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 34f17ebf-3db3-42bf-af14-e93e366ee8be kind: Event lastTimestamp: "2026-04-24T21:30:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:29:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:53Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz.18a9684159191c98 namespace: kserve-ci-e2e-test resourceVersion: "16883" uid: 61d271a0-fb3f-4e63-9dc2-c5f6f6a321dc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 34f17ebf-3db3-42bf-af14-e93e366ee8be kind: Event lastTimestamp: "2026-04-24T21:30:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:30:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:36Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz.18a968538ca227e3 namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: 1df6746b-e95c-4505-aa26-626ecc0b9e84 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 34f17ebf-3db3-42bf-af14-e93e366ee8be kind: Event lastTimestamp: "2026-04-24T21:30:36Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-24T21:30:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:36Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz.18a968538ca270c1 namespace: kserve-ci-e2e-test resourceVersion: "16689" uid: ed4177ea-f3bc-4f38-92c4-e7c7ddc36e33 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 34f17ebf-3db3-42bf-af14-e93e366ee8be kind: Event lastTimestamp: "2026-04-24T21:30:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:30:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:36Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz.18a968538ca2a802 namespace: kserve-ci-e2e-test resourceVersion: "16692" uid: 9e4285ce-5f12-4508-b2fa-fd5b0f7b296e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:30:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 34f17ebf-3db3-42bf-af14-e93e366ee8be kind: Event lastTimestamp: "2026-04-24T21:30:53Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:30:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:53Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz.18a96854353ee89b namespace: kserve-ci-e2e-test resourceVersion: "16881" uid: c1243677-a4b0-4fa1-95b1-2222b2ec9d3c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-66937-predictor-56b7d5bf78 namespace: kserve-ci-e2e-test resourceVersion: "15969" uid: ce7e0da5-8a85-4d18-81bb-207aa5daabf4 kind: Event lastTimestamp: "2026-04-24T21:29:10Z" message: 'Created pod: isvc-logger-raw-66937-predictor-56b7d5bf78-2nrcz' metadata: creationTimestamp: "2026-04-24T21:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:29:10Z" name: isvc-logger-raw-66937-predictor-56b7d5bf78.18a9683fb5c20257 namespace: kserve-ci-e2e-test resourceVersion: "15981" uid: f0861cf8-ee31-47de-b7e9-5fb65ea13618 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-66937-predictor namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 47d57640-b023-4b3c-a023-14a629b056cc kind: Event lastTimestamp: "2026-04-24T21:29:10Z" message: Scaled up replica set isvc-logger-raw-66937-predictor-56b7d5bf78 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:29:10Z" name: isvc-logger-raw-66937-predictor.18a9683fb2451aa5 namespace: kserve-ci-e2e-test resourceVersion: "15971" uid: 89551c27-b507-4b11-ad8b-f4af44782dd6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:29:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-66937-predictor namespace: kserve-ci-e2e-test resourceVersion: "15968" uid: 605d55b3-5876-4bce-bfd9-b21d0e01c148 kind: Event lastTimestamp: "2026-04-24T21:30:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:29:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:10Z" name: isvc-logger-raw-66937-predictor.18a9684330dbe25c namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: 889a89b7-ba24-44cc-b8e9-d7a2122522c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:29:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-66937-predictor namespace: kserve-ci-e2e-test resourceVersion: "15968" uid: 605d55b3-5876-4bce-bfd9-b21d0e01c148 kind: Event lastTimestamp: "2026-04-24T21:30:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:29:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:10Z" name: isvc-logger-raw-66937-predictor.18a96843314d5cd7 namespace: kserve-ci-e2e-test resourceVersion: "16468" uid: f927e108-c448-4a0b-bf18-d883be7d127b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-66937-predictor namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 605d55b3-5876-4bce-bfd9-b21d0e01c148 kind: Event lastTimestamp: "2026-04-24T21:30:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:30:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:25Z" name: isvc-logger-raw-66937-predictor.18a968512cdad9c2 namespace: kserve-ci-e2e-test resourceVersion: "16573" uid: 911323a0-5045-4682-83f8-cef2a32eb98a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-66937-predictor namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 605d55b3-5876-4bce-bfd9-b21d0e01c148 kind: Event lastTimestamp: "2026-04-24T21:30:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:30:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:25Z" name: isvc-logger-raw-66937-predictor.18a968512d3d880a namespace: kserve-ci-e2e-test resourceVersion: "16575" uid: bf59fc70-2ce2-4a6d-9ece-d747de476d02 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T21:29:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-66937 namespace: kserve-ci-e2e-test resourceVersion: "15957" uid: 42a8d855-adf4-4f19-9759-7ff7c634672b kind: Event lastTimestamp: "2026-04-24T21:30:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:30:23Z" name: isvc-logger-raw-66937.18a9683fb21e6605 namespace: kserve-ci-e2e-test resourceVersion: "16563" uid: d84584f5-7187-43e4-b0a6-139d33f0b56c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:29:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-66937 namespace: kserve-ci-e2e-test resourceVersion: "15957" uid: 42a8d855-adf4-4f19-9759-7ff7c634672b kind: Event lastTimestamp: "2026-04-24T21:29:10Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-66937": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-66937": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:29:10Z" name: isvc-logger-raw-66937.18a9683fb29a127c namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: 3fdcf485-7fed-4f8c-bd80-badf532f1fe5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-66937 namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 42a8d855-adf4-4f19-9759-7ff7c634672b kind: Event lastTimestamp: "2026-04-24T21:29:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-66937": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:29:10Z" name: isvc-logger-raw-66937.18a9683fb8022d39 namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: 7cd185ed-7e1f-4480-a967-53a8b3ba0295 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-66937 namespace: kserve-ci-e2e-test resourceVersion: "16559" uid: 42a8d855-adf4-4f19-9759-7ff7c634672b kind: Event lastTimestamp: "2026-04-24T21:30:23Z" message: InferenceService [isvc-logger-raw-66937] is Ready metadata: creationTimestamp: "2026-04-24T21:30:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:30:23Z" name: isvc-logger-raw-66937.18a96850bcb4d7f9 namespace: kserve-ci-e2e-test resourceVersion: "16560" uid: 23cbcf48-e4a6-40d9-ad73-3d37ab5f8f72 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:33:06.372412Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd namespace: kserve-ci-e2e-test resourceVersion: "17874" uid: 63d2248f-f7e9-4ed0-b369-0d40a0b04454 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-8d5709-predictor-59cbfc9977-bkkhd to ip-10-0-132-118.ec2.internal metadata: creationTimestamp: "2026-04-24T21:33:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:33:06Z" name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd.18a968768e03e2db namespace: kserve-ci-e2e-test resourceVersion: "17878" uid: dde22e98-1f5c-4761-b96b-0d02f849cb19 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6b5fbb9956-pzrgr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:33:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd namespace: kserve-ci-e2e-test resourceVersion: "17883" uid: 63d2248f-f7e9-4ed0-b369-0d40a0b04454 kind: Event lastTimestamp: "2026-04-24T21:33:06Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:33:06Z" name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd.18a96876a6a5c2c3 namespace: kserve-ci-e2e-test resourceVersion: "17892" uid: c4df5102-1421-4d67-99e1-7fdc0ef47baa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:33:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd namespace: kserve-ci-e2e-test resourceVersion: "17876" uid: 63d2248f-f7e9-4ed0-b369-0d40a0b04454 kind: Event lastTimestamp: "2026-04-24T21:33:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:85f85c16414082de64de782f602228652c10c5bace08b518e803a7e911c62e5e" already present on machine metadata: creationTimestamp: "2026-04-24T21:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:33:06Z" name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd.18a96876a8484e2b namespace: kserve-ci-e2e-test resourceVersion: "17894" uid: 292ce79b-27eb-4e46-8d07-015af4204e88 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:33:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd namespace: kserve-ci-e2e-test resourceVersion: "17876" uid: 63d2248f-f7e9-4ed0-b369-0d40a0b04454 kind: Event lastTimestamp: "2026-04-24T21:33:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:33:06Z" name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd.18a96876ab6d296d namespace: kserve-ci-e2e-test resourceVersion: "17896" uid: e0ea72e2-a93b-4903-aeb5-c299139ef6c9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:33:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd namespace: kserve-ci-e2e-test resourceVersion: "17876" uid: 63d2248f-f7e9-4ed0-b369-0d40a0b04454 kind: Event lastTimestamp: "2026-04-24T21:33:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:33:06Z" name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd.18a96876ac1747ae namespace: kserve-ci-e2e-test resourceVersion: "17897" uid: fc2dbe81-ea93-4a0c-9fb4-b302b4b408de reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:33:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd namespace: kserve-ci-e2e-test resourceVersion: "17876" uid: 63d2248f-f7e9-4ed0-b369-0d40a0b04454 kind: Event lastTimestamp: "2026-04-24T21:33:11Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:33:11Z" name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd.18a96877c2fa0c44 namespace: kserve-ci-e2e-test resourceVersion: "17940" uid: cc421f67-ad2f-46bd-bd54-28e97ef3f28f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:33:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd namespace: kserve-ci-e2e-test resourceVersion: "17876" uid: 63d2248f-f7e9-4ed0-b369-0d40a0b04454 kind: Event lastTimestamp: "2026-04-24T21:33:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:33:11Z" name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd.18a96877c5bca022 namespace: kserve-ci-e2e-test resourceVersion: "17943" uid: 6694c88f-0579-4216-9caf-e77332b965e6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:33:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd namespace: kserve-ci-e2e-test resourceVersion: "17876" uid: 63d2248f-f7e9-4ed0-b369-0d40a0b04454 kind: Event lastTimestamp: "2026-04-24T21:33:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:33:11Z" name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd.18a96877c6328e1b namespace: kserve-ci-e2e-test resourceVersion: "17946" uid: c6d203fa-247a-4828-8690-ba860274c985 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:33:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd namespace: kserve-ci-e2e-test resourceVersion: "17876" uid: 63d2248f-f7e9-4ed0-b369-0d40a0b04454 kind: Event lastTimestamp: "2026-04-24T21:33:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:33:11Z" name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd.18a96877c641003f namespace: kserve-ci-e2e-test resourceVersion: "17947" uid: a43303ba-54e6-4e5b-a34c-13611f565749 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:33:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd namespace: kserve-ci-e2e-test resourceVersion: "17876" uid: 63d2248f-f7e9-4ed0-b369-0d40a0b04454 kind: Event lastTimestamp: "2026-04-24T21:33:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:33:11Z" name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd.18a96877c9fdf607 namespace: kserve-ci-e2e-test resourceVersion: "17949" uid: 2044e98a-012b-4ecf-ac7e-ed354c2579ad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:33:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd namespace: kserve-ci-e2e-test resourceVersion: "17876" uid: 63d2248f-f7e9-4ed0-b369-0d40a0b04454 kind: Event lastTimestamp: "2026-04-24T21:33:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:33:11Z" name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd.18a96877ca6ba8f7 namespace: kserve-ci-e2e-test resourceVersion: "17950" uid: c61b0cb0-587c-48ab-a89a-abbf6ae8d365 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T21:33:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd namespace: kserve-ci-e2e-test resourceVersion: "17876" uid: 63d2248f-f7e9-4ed0-b369-0d40a0b04454 kind: Event lastTimestamp: "2026-04-24T21:34:09Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:33:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:09Z" name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd.18a968783aad6cd6 namespace: kserve-ci-e2e-test resourceVersion: "18368" uid: 045c6f5a-5764-473f-8c06-27509ce57057 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd namespace: kserve-ci-e2e-test resourceVersion: "17876" uid: 63d2248f-f7e9-4ed0-b369-0d40a0b04454 kind: Event lastTimestamp: "2026-04-24T21:34:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:34:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:44Z" name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd.18a9688d6c11487b namespace: kserve-ci-e2e-test resourceVersion: "18731" uid: 21bc0fb4-6822-429c-9a08-b9d93f66a333 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd namespace: kserve-ci-e2e-test resourceVersion: "17876" uid: 63d2248f-f7e9-4ed0-b369-0d40a0b04454 kind: Event lastTimestamp: "2026-04-24T21:34:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:34:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:44Z" name: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd.18a9688d6c11ce4f namespace: kserve-ci-e2e-test resourceVersion: "18734" uid: 958e2485-3ceb-4feb-8573-e70c527068cd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:33:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-8d5709-predictor-59cbfc9977 namespace: kserve-ci-e2e-test resourceVersion: "17864" uid: 4e658aef-e237-4983-8306-3ab9e876a9c4 kind: Event lastTimestamp: "2026-04-24T21:33:06Z" message: 'Created pod: isvc-primary-8d5709-predictor-59cbfc9977-bkkhd' metadata: creationTimestamp: "2026-04-24T21:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:33:06Z" name: isvc-primary-8d5709-predictor-59cbfc9977.18a968768d9254c9 namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: e947bca4-fb83-4a8d-9018-01ec0f2db1cf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:33:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-8d5709-predictor namespace: kserve-ci-e2e-test resourceVersion: "17862" uid: 1f8e5e86-75a7-47e0-b2c6-89c284e500e6 kind: Event lastTimestamp: "2026-04-24T21:33:06Z" message: Scaled up replica set isvc-primary-8d5709-predictor-59cbfc9977 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:33:06Z" name: isvc-primary-8d5709-predictor.18a96876886b7cbe namespace: kserve-ci-e2e-test resourceVersion: "17866" uid: 952a448f-149f-4ff3-990e-b580947e88b3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:33:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-8d5709-predictor namespace: kserve-ci-e2e-test resourceVersion: "17863" uid: cac1171b-7918-437e-8db0-e15b79c9014e kind: Event lastTimestamp: "2026-04-24T21:33:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:33:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:33:51Z" name: isvc-primary-8d5709-predictor.18a9687a07753ac0 namespace: kserve-ci-e2e-test resourceVersion: "18237" uid: 1e021eca-cf1e-418b-8189-8594fa0d5dd9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:33:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-8d5709-predictor namespace: kserve-ci-e2e-test resourceVersion: "17863" uid: cac1171b-7918-437e-8db0-e15b79c9014e kind: Event lastTimestamp: "2026-04-24T21:33:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:33:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:33:51Z" name: isvc-primary-8d5709-predictor.18a9687a07d8de54 namespace: kserve-ci-e2e-test resourceVersion: "18238" uid: f33ee2d9-0b92-4042-93af-cf4c54f72dc8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:33:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-8d5709-predictor namespace: kserve-ci-e2e-test resourceVersion: "18021" uid: cac1171b-7918-437e-8db0-e15b79c9014e kind: Event lastTimestamp: "2026-04-24T21:34:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:36Z" name: isvc-primary-8d5709-predictor.18a9687d868241ce namespace: kserve-ci-e2e-test resourceVersion: "18624" uid: 872787f5-9301-43c7-8994-55a98ad398fb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:33:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-8d5709-predictor namespace: kserve-ci-e2e-test resourceVersion: "18021" uid: cac1171b-7918-437e-8db0-e15b79c9014e kind: Event lastTimestamp: "2026-04-24T21:34:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:36Z" name: isvc-primary-8d5709-predictor.18a9687d86e22169 namespace: kserve-ci-e2e-test resourceVersion: "18625" uid: 1391eb22-aeac-4890-83d2-5f469bbf93c7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-24T21:33:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-8d5709 namespace: kserve-ci-e2e-test resourceVersion: "17832" uid: 9b8b9666-fe5e-4a6b-9cf9-7f4f24bcbcce kind: Event lastTimestamp: "2026-04-24T21:34:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:34:19Z" name: isvc-primary-8d5709.18a96876882b0d2a namespace: kserve-ci-e2e-test resourceVersion: "18440" uid: 0fab411e-edd1-4811-8076-f4fc2d8d190c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:33:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-8d5709 namespace: kserve-ci-e2e-test resourceVersion: "17832" uid: 9b8b9666-fe5e-4a6b-9cf9-7f4f24bcbcce kind: Event lastTimestamp: "2026-04-24T21:33:06Z" message: 'Failed to update status for InferenceService "isvc-primary-8d5709": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-8d5709": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:33:06Z" name: isvc-primary-8d5709.18a9687688a48a74 namespace: kserve-ci-e2e-test resourceVersion: "17868" uid: e3714a82-0e4f-4d2f-bb7b-46fbbdba3cc9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:33:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-8d5709 namespace: kserve-ci-e2e-test resourceVersion: "17832" uid: 9b8b9666-fe5e-4a6b-9cf9-7f4f24bcbcce kind: Event lastTimestamp: "2026-04-24T21:33:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-8d5709": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:33:06Z" name: isvc-primary-8d5709.18a9687688a4e9d9 namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 4a24ff23-776f-4c36-abf0-0e932e1d3dfa reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-8d5709 namespace: kserve-ci-e2e-test resourceVersion: "18438" uid: 9b8b9666-fe5e-4a6b-9cf9-7f4f24bcbcce kind: Event lastTimestamp: "2026-04-24T21:34:19Z" message: InferenceService [isvc-primary-8d5709] is Ready metadata: creationTimestamp: "2026-04-24T21:34:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:34:19Z" name: isvc-primary-8d5709.18a968879dd79329 namespace: kserve-ci-e2e-test resourceVersion: "18439" uid: eaea9e1d-8963-4f8d-bed8-f6f063bee800 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:23:34.972983Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12875" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j to ip-10-0-132-118.ec2.internal metadata: creationTimestamp: "2026-04-24T21:23:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:23:34Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a967f183f48360 namespace: kserve-ci-e2e-test resourceVersion: "12880" uid: 4992f528-e1ad-42c6-a45a-065b922ff843 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6b5fbb9956-pzrgr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:23:35Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12887" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:23:35Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:23:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:23:35Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a967f19d02969f namespace: kserve-ci-e2e-test resourceVersion: "12898" uid: c25d8e0c-b262-41da-b464-5ac589c9c7cf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:23:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:23:35Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:85f85c16414082de64de782f602228652c10c5bace08b518e803a7e911c62e5e" metadata: creationTimestamp: "2026-04-24T21:23:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:23:35Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a967f19e8f1a46 namespace: kserve-ci-e2e-test resourceVersion: "12900" uid: 534e76ed-5dd5-4cca-8019-fd3720661937 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:23:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:23:39Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:85f85c16414082de64de782f602228652c10c5bace08b518e803a7e911c62e5e" in 3.759s (3.759s including waiting). Image size: 299849144 bytes.' metadata: creationTimestamp: "2026-04-24T21:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:23:39Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a967f27e9e8309 namespace: kserve-ci-e2e-test resourceVersion: "12927" uid: 5b122ca1-0675-4ee8-844c-53aed26002a2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:23:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:23:39Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:23:39Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a967f281bbf658 namespace: kserve-ci-e2e-test resourceVersion: "12929" uid: 7b562e8e-1746-4812-871d-fd660cc80cf3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:23:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:23:39Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:23:39Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a967f2823206bb namespace: kserve-ci-e2e-test resourceVersion: "12930" uid: 3168fe48-6589-4e5c-87d3-1d242470bd04 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:23:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:23:43Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1447" metadata: creationTimestamp: "2026-04-24T21:23:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:23:43Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a967f390a8a795 namespace: kserve-ci-e2e-test resourceVersion: "12975" uid: 34a49155-6485-4072-9100-943b8405fc20 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:23:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:23:57Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1447" in 13.451s (13.451s including waiting). Image size: 1560926130 bytes.' metadata: creationTimestamp: "2026-04-24T21:23:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:23:57Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a967f6b26cc3b1 namespace: kserve-ci-e2e-test resourceVersion: "13070" uid: 58a23623-62ce-4ad1-8869-7f9aab527394 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:23:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:23:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:23:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:23:57Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a967f6b666ee9c namespace: kserve-ci-e2e-test resourceVersion: "13071" uid: bc235660-1037-406e-ae9d-7a2bf41b0d44 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:23:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:23:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:23:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:23:57Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a967f6b6d1f7fe namespace: kserve-ci-e2e-test resourceVersion: "13072" uid: 855b6191-c0f9-4d7f-80de-a7c42f629341 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:23:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:23:57Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T21:23:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:23:57Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a967f6b6def426 namespace: kserve-ci-e2e-test resourceVersion: "13073" uid: 377dc20f-0d6d-4185-a21d-1199996bb887 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:23:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:23:59Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.44s (2.44s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T21:23:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:23:59Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a967f7485ac0d7 namespace: kserve-ci-e2e-test resourceVersion: "13090" uid: c3efd149-61ff-4079-937d-d844df7b42da reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:23:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:23:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:23:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:23:59Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a967f74b1b6e9d namespace: kserve-ci-e2e-test resourceVersion: "13092" uid: 072a6539-5966-4a30-b130-57f8505bb5e3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:23:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:23:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:23:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:23:59Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a967f74b9fd915 namespace: kserve-ci-e2e-test resourceVersion: "13093" uid: a8269c70-e85d-4c69-b3db-3ca3863a0710 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:23:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:23:59Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:e6428204baa077dfe80627f4a0f194329877d007e326693ea30a582520538a96" metadata: creationTimestamp: "2026-04-24T21:23:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:23:59Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a967f74bab3b5d namespace: kserve-ci-e2e-test resourceVersion: "13094" uid: f1d528c5-355b-4508-a1d5-3c4b13373558 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:24:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:24:02Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:e6428204baa077dfe80627f4a0f194329877d007e326693ea30a582520538a96" in 2.528s (2.528s including waiting). Image size: 238045275 bytes.' metadata: creationTimestamp: "2026-04-24T21:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:24:02Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a967f7e268e57b namespace: kserve-ci-e2e-test resourceVersion: "13113" uid: a179a89e-6537-4edb-a646-5c04eb87ff41 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:24:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:24:02Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-24T21:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:24:02Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a967f7e5e382bd namespace: kserve-ci-e2e-test resourceVersion: "13115" uid: 05f1467e-5241-41b7-baea-e94458bcf25e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:24:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:24:02Z" message: Started container agent metadata: creationTimestamp: "2026-04-24T21:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:24:02Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a967f7e6667270 namespace: kserve-ci-e2e-test resourceVersion: "13116" uid: d6ddad5a-71d5-4d4c-9573-a267d0866c84 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-24T21:24:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:25:35Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:35Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a967f80161b136 namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 5679824a-b322-4dc0-8740-5d69e7c9eb5a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-24T21:24:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:25:35Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:35Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a967f801688ac0 namespace: kserve-ci-e2e-test resourceVersion: "13943" uid: d2ea96ba-9694-4309-b8b5-83d399dae41a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:25:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:20Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a9680a02a9c66b namespace: kserve-ci-e2e-test resourceVersion: "13716" uid: cf662b8c-fd2a-4643-81f4-2667d85b3f13 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:25:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:20Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a9680a02ada2d1 namespace: kserve-ci-e2e-test resourceVersion: "13718" uid: ace26aaf-1232-458c-a153-0ae64d5c2453 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:25:20Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:20Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a9680a02b0f238 namespace: kserve-ci-e2e-test resourceVersion: "13720" uid: 24c611d9-39c4-4242-ad68-925d0da0263e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:25:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: d42a2b82-77ac-4ab0-9ab3-c2f5ea69b8be kind: Event lastTimestamp: "2026-04-24T21:25:34Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:25:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:34Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j.18a9680b194d1caf namespace: kserve-ci-e2e-test resourceVersion: "13927" uid: 9e4b8411-e31f-43f5-846f-dd9f06838291 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:23:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6 namespace: kserve-ci-e2e-test resourceVersion: "12868" uid: 4d4a0389-45ad-4d32-a43e-e08824715ca6 kind: Event lastTimestamp: "2026-04-24T21:23:34Z" message: 'Created pod: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6-rv67j' metadata: creationTimestamp: "2026-04-24T21:23:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:23:34Z" name: isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6.18a967f1838dace1 namespace: kserve-ci-e2e-test resourceVersion: "12876" uid: fa188f6d-cfe3-43eb-a181-f607bb27ec96 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:23:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-3062b-predictor namespace: kserve-ci-e2e-test resourceVersion: "12867" uid: 100ed58c-9e94-4275-a644-7acdbdf0a127 kind: Event lastTimestamp: "2026-04-24T21:23:34Z" message: Scaled up replica set isvc-raw-sklearn-batcher-3062b-predictor-5b6b6b84f6 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:23:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:23:34Z" name: isvc-raw-sklearn-batcher-3062b-predictor.18a967f17e33b6fa namespace: kserve-ci-e2e-test resourceVersion: "12870" uid: a724bae1-e754-4c78-87d3-045fc04e149a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:23:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-3062b-predictor namespace: kserve-ci-e2e-test resourceVersion: "12879" uid: f75190d1-c446-4175-b459-ee7b2059ac2b kind: Event lastTimestamp: "2026-04-24T21:24:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:24:35Z" name: isvc-raw-sklearn-batcher-3062b-predictor.18a967f504659484 namespace: kserve-ci-e2e-test resourceVersion: "13364" uid: 3b6740ff-257e-447b-8d81-b1324fbc5f11 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:23:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-3062b-predictor namespace: kserve-ci-e2e-test resourceVersion: "12879" uid: f75190d1-c446-4175-b459-ee7b2059ac2b kind: Event lastTimestamp: "2026-04-24T21:24:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:24:35Z" name: isvc-raw-sklearn-batcher-3062b-predictor.18a967f505043140 namespace: kserve-ci-e2e-test resourceVersion: "13365" uid: 0b9de542-86ce-4624-ad14-fe59c1044eee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:24:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-3062b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13017" uid: f75190d1-c446-4175-b459-ee7b2059ac2b kind: Event lastTimestamp: "2026-04-24T21:25:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:24:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:25:05Z" name: isvc-raw-sklearn-batcher-3062b-predictor.18a967fc02e986b6 namespace: kserve-ci-e2e-test resourceVersion: "13575" uid: df882d77-f053-4765-883e-5e5e71ab862f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:24:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-3062b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13017" uid: f75190d1-c446-4175-b459-ee7b2059ac2b kind: Event lastTimestamp: "2026-04-24T21:25:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:24:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:25:05Z" name: isvc-raw-sklearn-batcher-3062b-predictor.18a967fc03687a29 namespace: kserve-ci-e2e-test resourceVersion: "13576" uid: 0d58c869-62c4-4000-b249-cd1276c72d4a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-3062b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: f75190d1-c446-4175-b459-ee7b2059ac2b kind: Event lastTimestamp: "2026-04-24T21:25:20Z" message: deployments.apps "isvc-raw-sklearn-batcher-3062b-predictor" not found metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:25:20Z" name: isvc-raw-sklearn-batcher-3062b-predictor.18a9680a004e4cd8 namespace: kserve-ci-e2e-test resourceVersion: "13696" uid: 3927fd57-afd7-497d-8d14-d75ba745bc19 reason: FailedGetScale reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-3062b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: f75190d1-c446-4175-b459-ee7b2059ac2b kind: Event lastTimestamp: "2026-04-24T21:25:20Z" message: 'Operation cannot be fulfilled on horizontalpodautoscalers.autoscaling "isvc-raw-sklearn-batcher-3062b-predictor": StorageError: invalid object, Code: 4, Key: /kubernetes.io/horizontalpodautoscalers/kserve-ci-e2e-test/isvc-raw-sklearn-batcher-3062b-predictor, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: f75190d1-c446-4175-b459-ee7b2059ac2b, UID in object meta: ' metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:25:20Z" name: isvc-raw-sklearn-batcher-3062b-predictor.18a9680a009ae64d namespace: kserve-ci-e2e-test resourceVersion: "13700" uid: b44e5efb-d287-4e43-9f3e-d4ce38f5a3dd reason: FailedUpdateStatus reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-24T21:23:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-3062b namespace: kserve-ci-e2e-test resourceVersion: "12854" uid: 574542b9-3be4-4c0c-beda-f7408856623a kind: Event lastTimestamp: "2026-04-24T21:25:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:23:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:25:05Z" name: isvc-raw-sklearn-batcher-3062b.18a967f1846cc38c namespace: kserve-ci-e2e-test resourceVersion: "13590" uid: 6071c8d3-ee7a-4000-90ae-06000dc4af77 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:23:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-3062b namespace: kserve-ci-e2e-test resourceVersion: "12854" uid: 574542b9-3be4-4c0c-beda-f7408856623a kind: Event lastTimestamp: "2026-04-24T21:23:34Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-3062b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-3062b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:23:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:23:34Z" name: isvc-raw-sklearn-batcher-3062b.18a967f184e3a7fc namespace: kserve-ci-e2e-test resourceVersion: "12886" uid: 1dc67a44-0dda-4b00-91d3-fb12287f7b9e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:23:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-3062b namespace: kserve-ci-e2e-test resourceVersion: "12854" uid: 574542b9-3be4-4c0c-beda-f7408856623a kind: Event lastTimestamp: "2026-04-24T21:23:34Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-3062b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:23:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:23:34Z" name: isvc-raw-sklearn-batcher-3062b.18a967f184e404db namespace: kserve-ci-e2e-test resourceVersion: "12889" uid: 4abb8ff3-620b-4c8c-ab8a-684a61a7684d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-3062b namespace: kserve-ci-e2e-test resourceVersion: "13587" uid: 574542b9-3be4-4c0c-beda-f7408856623a kind: Event lastTimestamp: "2026-04-24T21:25:05Z" message: InferenceService [isvc-raw-sklearn-batcher-3062b] is Ready metadata: creationTimestamp: "2026-04-24T21:25:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:25:05Z" name: isvc-raw-sklearn-batcher-3062b.18a96806b28b442f namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 5ef133e7-eb0b-422d-a32a-acdc2d109696 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:34:26.483284Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-8d5709-predictor-67d976f9d6-pcqwt namespace: kserve-ci-e2e-test resourceVersion: "18519" uid: db2529bd-999e-40c2-9d81-0ea7ce55fa8d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-8d5709-predictor-67d976f9d6-pcqwt to ip-10-0-132-118.ec2.internal metadata: creationTimestamp: "2026-04-24T21:34:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:34:26Z" name: isvc-secondary-8d5709-predictor-67d976f9d6-pcqwt.18a9688934fed448 namespace: kserve-ci-e2e-test resourceVersion: "18523" uid: 1e9d5e42-67cc-48e5-b790-41c3d008847c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6b5fbb9956-pzrgr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:26Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-8d5709-predictor-67d976f9d6-pcqwt namespace: kserve-ci-e2e-test resourceVersion: "18527" uid: db2529bd-999e-40c2-9d81-0ea7ce55fa8d kind: Event lastTimestamp: "2026-04-24T21:34:26Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:34:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:34:26Z" name: isvc-secondary-8d5709-predictor-67d976f9d6-pcqwt.18a968894e06ed40 namespace: kserve-ci-e2e-test resourceVersion: "18535" uid: b3763550-60ca-4278-abd3-a0759d5c8d0b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:34:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-8d5709-predictor-67d976f9d6-pcqwt namespace: kserve-ci-e2e-test resourceVersion: "18521" uid: db2529bd-999e-40c2-9d81-0ea7ce55fa8d kind: Event lastTimestamp: "2026-04-24T21:34:33Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:85f85c16414082de64de782f602228652c10c5bace08b518e803a7e911c62e5e" already present on machine metadata: creationTimestamp: "2026-04-24T21:34:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:33Z" name: isvc-secondary-8d5709-predictor-67d976f9d6-pcqwt.18a968894f5b035d namespace: kserve-ci-e2e-test resourceVersion: "18597" uid: 6965374e-c0d5-481d-a5d0-5c4d245d7359 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:34:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-8d5709-predictor-67d976f9d6-pcqwt namespace: kserve-ci-e2e-test resourceVersion: "18521" uid: db2529bd-999e-40c2-9d81-0ea7ce55fa8d kind: Event lastTimestamp: "2026-04-24T21:34:33Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:34:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:33Z" name: isvc-secondary-8d5709-predictor-67d976f9d6-pcqwt.18a9688951fd8b4c namespace: kserve-ci-e2e-test resourceVersion: "18600" uid: 4edf9a37-173c-48f0-ae0e-f74fb7becfd9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:34:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-8d5709-predictor-67d976f9d6-pcqwt namespace: kserve-ci-e2e-test resourceVersion: "18521" uid: db2529bd-999e-40c2-9d81-0ea7ce55fa8d kind: Event lastTimestamp: "2026-04-24T21:34:33Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:34:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:33Z" name: isvc-secondary-8d5709-predictor-67d976f9d6-pcqwt.18a96889526b61d1 namespace: kserve-ci-e2e-test resourceVersion: "18602" uid: 9d536e7b-f9d1-42be-8019-fa8344e6406a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-8d5709-predictor-67d976f9d6-pcqwt namespace: kserve-ci-e2e-test resourceVersion: "18521" uid: db2529bd-999e-40c2-9d81-0ea7ce55fa8d kind: Event lastTimestamp: "2026-04-24T21:34:38Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-8d5709-predictor-67d976f9d6-pcqwt_kserve-ci-e2e-test(db2529bd-999e-40c2-9d81-0ea7ce55fa8d) metadata: creationTimestamp: "2026-04-24T21:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:38Z" name: isvc-secondary-8d5709-predictor-67d976f9d6-pcqwt.18a9688c159ca6e5 namespace: kserve-ci-e2e-test resourceVersion: "18639" uid: 34fe4d11-db7f-4a5d-a7df-8d182f0c4733 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-8d5709-predictor-67d976f9d6 namespace: kserve-ci-e2e-test resourceVersion: "18510" uid: a9f7b3eb-ac49-4c26-b9fd-0bc8bb7a3b7e kind: Event lastTimestamp: "2026-04-24T21:34:26Z" message: 'Created pod: isvc-secondary-8d5709-predictor-67d976f9d6-pcqwt' metadata: creationTimestamp: "2026-04-24T21:34:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:26Z" name: isvc-secondary-8d5709-predictor-67d976f9d6.18a96889348fc80b namespace: kserve-ci-e2e-test resourceVersion: "18520" uid: 4b2f5e8d-9ff1-488b-bbb0-881e0efeed6c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-8d5709-predictor namespace: kserve-ci-e2e-test resourceVersion: "18508" uid: 83fb489a-9d7e-48f3-a3fa-669f1d55b9d3 kind: Event lastTimestamp: "2026-04-24T21:34:26Z" message: Scaled up replica set isvc-secondary-8d5709-predictor-67d976f9d6 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:34:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:26Z" name: isvc-secondary-8d5709-predictor.18a96889311c68ea namespace: kserve-ci-e2e-test resourceVersion: "18512" uid: 6742e825-1db9-497d-a7ae-eb68a6914a50 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-8d5709-predictor namespace: kserve-ci-e2e-test resourceVersion: "18509" uid: e54cb761-e1bd-4a0d-af99-30035672b2fe kind: Event lastTimestamp: "2026-04-24T21:34:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:41Z" name: isvc-secondary-8d5709-predictor.18a9688caff26c00 namespace: kserve-ci-e2e-test resourceVersion: "18659" uid: 84103973-be0b-436d-b0ec-910669ad4c1d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-8d5709-predictor namespace: kserve-ci-e2e-test resourceVersion: "18509" uid: e54cb761-e1bd-4a0d-af99-30035672b2fe kind: Event lastTimestamp: "2026-04-24T21:34:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:41Z" name: isvc-secondary-8d5709-predictor.18a9688cb056ae34 namespace: kserve-ci-e2e-test resourceVersion: "18661" uid: 4095e50e-19aa-459c-8618-6ccc6d34c020 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-24T21:34:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-8d5709 namespace: kserve-ci-e2e-test resourceVersion: "18499" uid: 6af9cf40-f095-4799-b326-db2daefa5be7 kind: Event lastTimestamp: "2026-04-24T21:34:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:34:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:34:40Z" name: isvc-secondary-8d5709.18a9688930f403a2 namespace: kserve-ci-e2e-test resourceVersion: "18652" uid: 045e5c8d-26f6-496b-85cb-f8fc6629d568 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-8d5709 namespace: kserve-ci-e2e-test resourceVersion: "18499" uid: 6af9cf40-f095-4799-b326-db2daefa5be7 kind: Event lastTimestamp: "2026-04-24T21:34:26Z" message: 'Failed to update status for InferenceService "isvc-secondary-8d5709": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-8d5709": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:34:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:34:26Z" name: isvc-secondary-8d5709.18a9688931c67b9b namespace: kserve-ci-e2e-test resourceVersion: "18514" uid: f9e8bf3b-a0dc-4a62-93d8-791039b74ab5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-8d5709 namespace: kserve-ci-e2e-test resourceVersion: "18499" uid: 6af9cf40-f095-4799-b326-db2daefa5be7 kind: Event lastTimestamp: "2026-04-24T21:34:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-8d5709": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:34:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:34:26Z" name: isvc-secondary-8d5709.18a9688931c6da04 namespace: kserve-ci-e2e-test resourceVersion: "18516" uid: 5a0d8c50-9449-4643-88f8-6601f3ea8920 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:25:20.335268Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn namespace: kserve-ci-e2e-test resourceVersion: "13739" uid: d35a9f7d-8a52-4cc0-a507-a704d6a2a614 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn to ip-10-0-132-118.ec2.internal metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:25:20Z" name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn.18a9680a0c097089 namespace: kserve-ci-e2e-test resourceVersion: "13750" uid: 689350e4-7d9e-485a-9da7-5f15292ceaa6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6b5fbb9956-pzrgr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn namespace: kserve-ci-e2e-test resourceVersion: "13758" uid: d35a9f7d-8a52-4cc0-a507-a704d6a2a614 kind: Event lastTimestamp: "2026-04-24T21:25:20Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:25:20Z" name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn.18a9680a2522ee86 namespace: kserve-ci-e2e-test resourceVersion: "13781" uid: 8a1f8bd1-5e78-4592-a943-a8df9846b59c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn namespace: kserve-ci-e2e-test resourceVersion: "13744" uid: d35a9f7d-8a52-4cc0-a507-a704d6a2a614 kind: Event lastTimestamp: "2026-04-24T21:25:20Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:85f85c16414082de64de782f602228652c10c5bace08b518e803a7e911c62e5e" already present on machine metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:20Z" name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn.18a9680a26c666e2 namespace: kserve-ci-e2e-test resourceVersion: "13783" uid: 1ff5c26b-9186-40d4-9e54-30ae6cca493a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn namespace: kserve-ci-e2e-test resourceVersion: "13744" uid: d35a9f7d-8a52-4cc0-a507-a704d6a2a614 kind: Event lastTimestamp: "2026-04-24T21:25:20Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:20Z" name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn.18a9680a29d8f91a namespace: kserve-ci-e2e-test resourceVersion: "13784" uid: 1185a781-affa-4fb3-8e64-975640dd5005 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn namespace: kserve-ci-e2e-test resourceVersion: "13744" uid: d35a9f7d-8a52-4cc0-a507-a704d6a2a614 kind: Event lastTimestamp: "2026-04-24T21:25:20Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:20Z" name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn.18a9680a2a526a73 namespace: kserve-ci-e2e-test resourceVersion: "13785" uid: 20920c04-3ab5-45e0-b46b-4fa1f81b7876 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn namespace: kserve-ci-e2e-test resourceVersion: "13744" uid: d35a9f7d-8a52-4cc0-a507-a704d6a2a614 kind: Event lastTimestamp: "2026-04-24T21:25:25Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:25Z" name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn.18a9680b27a12dc4 namespace: kserve-ci-e2e-test resourceVersion: "13844" uid: 236fc8eb-0c44-4a19-9d2c-c94774c125aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn namespace: kserve-ci-e2e-test resourceVersion: "13744" uid: d35a9f7d-8a52-4cc0-a507-a704d6a2a614 kind: Event lastTimestamp: "2026-04-24T21:25:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:25Z" name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn.18a9680b2ac9fe5e namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: d6057369-69a7-43f8-a3f6-45508673656a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn namespace: kserve-ci-e2e-test resourceVersion: "13744" uid: d35a9f7d-8a52-4cc0-a507-a704d6a2a614 kind: Event lastTimestamp: "2026-04-24T21:25:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:25Z" name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn.18a9680b2b39b33c namespace: kserve-ci-e2e-test resourceVersion: "13852" uid: ea71e839-2bfe-41a3-b146-648d3d7139d7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn namespace: kserve-ci-e2e-test resourceVersion: "13744" uid: d35a9f7d-8a52-4cc0-a507-a704d6a2a614 kind: Event lastTimestamp: "2026-04-24T21:25:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:25Z" name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn.18a9680b2b44cd32 namespace: kserve-ci-e2e-test resourceVersion: "13853" uid: de599ce4-470d-4ff5-b2aa-a5f0c7c8c922 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn namespace: kserve-ci-e2e-test resourceVersion: "13744" uid: d35a9f7d-8a52-4cc0-a507-a704d6a2a614 kind: Event lastTimestamp: "2026-04-24T21:25:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:25Z" name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn.18a9680b2e1fbf5f namespace: kserve-ci-e2e-test resourceVersion: "13856" uid: dd72778c-2665-4eba-becf-b54bfe48c59c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn namespace: kserve-ci-e2e-test resourceVersion: "13744" uid: d35a9f7d-8a52-4cc0-a507-a704d6a2a614 kind: Event lastTimestamp: "2026-04-24T21:25:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:25Z" name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn.18a9680b2e8e8fc4 namespace: kserve-ci-e2e-test resourceVersion: "13857" uid: cef43903-df84-496f-85a5-7f33a3798eec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn namespace: kserve-ci-e2e-test resourceVersion: "13744" uid: d35a9f7d-8a52-4cc0-a507-a704d6a2a614 kind: Event lastTimestamp: "2026-04-24T21:27:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:12Z" name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn.18a9680b63f263a8 namespace: kserve-ci-e2e-test resourceVersion: "14860" uid: 26b7c06d-c00a-4383-b62c-d00d39158018 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn namespace: kserve-ci-e2e-test resourceVersion: "13744" uid: d35a9f7d-8a52-4cc0-a507-a704d6a2a614 kind: Event lastTimestamp: "2026-04-24T21:27:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:10Z" name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn.18a96823adb65171 namespace: kserve-ci-e2e-test resourceVersion: "14746" uid: 7a538407-1a27-4d4f-a52f-404d67398bb2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn namespace: kserve-ci-e2e-test resourceVersion: "13744" uid: d35a9f7d-8a52-4cc0-a507-a704d6a2a614 kind: Event lastTimestamp: "2026-04-24T21:27:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:10Z" name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn.18a96823adb6e718 namespace: kserve-ci-e2e-test resourceVersion: "14748" uid: 6b0d25b2-202d-4d63-97c8-44f09de7c25c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn namespace: kserve-ci-e2e-test resourceVersion: "13744" uid: d35a9f7d-8a52-4cc0-a507-a704d6a2a614 kind: Event lastTimestamp: "2026-04-24T21:27:12Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:12Z" name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn.18a968241237bdaf namespace: kserve-ci-e2e-test resourceVersion: "14859" uid: 47db1726-c69c-4cf4-9c1c-3777b2e2d96f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8 namespace: kserve-ci-e2e-test resourceVersion: "13721" uid: 8d66148f-15e4-48c2-9cc2-d88431e0d166 kind: Event lastTimestamp: "2026-04-24T21:25:20Z" message: 'Created pod: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8-7vthn' metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:25:20Z" name: isvc-sklearn-graph-raw-da412-predictor-86b6b578d8.18a9680a0b99ca32 namespace: kserve-ci-e2e-test resourceVersion: "13743" uid: c15de333-756c-4443-a0ed-185a6464ff83 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-da412-predictor namespace: kserve-ci-e2e-test resourceVersion: "13717" uid: 12b14f55-4d60-472b-9d43-043b11476192 kind: Event lastTimestamp: "2026-04-24T21:25:20Z" message: Scaled up replica set isvc-sklearn-graph-raw-da412-predictor-86b6b578d8 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:25:20Z" name: isvc-sklearn-graph-raw-da412-predictor.18a9680a0422c0a2 namespace: kserve-ci-e2e-test resourceVersion: "13723" uid: 977eb2c6-e9b3-4be0-afbd-cad5b26734a1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:25:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-da412-predictor namespace: kserve-ci-e2e-test resourceVersion: "13719" uid: 4328ccc5-de01-47a1-aa7a-b23871698481 kind: Event lastTimestamp: "2026-04-24T21:26:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:26:05Z" name: isvc-sklearn-graph-raw-da412-predictor.18a9680d827e8163 namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: d32b67a5-48ca-444b-a23f-e4ee0761d9ad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:25:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-da412-predictor namespace: kserve-ci-e2e-test resourceVersion: "13719" uid: 4328ccc5-de01-47a1-aa7a-b23871698481 kind: Event lastTimestamp: "2026-04-24T21:26:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:26:05Z" name: isvc-sklearn-graph-raw-da412-predictor.18a9680d82dc2cee namespace: kserve-ci-e2e-test resourceVersion: "14172" uid: 0afac330-88f1-4b6e-9808-4db5570dfc1b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:25:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-da412-predictor namespace: kserve-ci-e2e-test resourceVersion: "13933" uid: 4328ccc5-de01-47a1-aa7a-b23871698481 kind: Event lastTimestamp: "2026-04-24T21:27:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:25:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:27:05Z" name: isvc-sklearn-graph-raw-da412-predictor.18a9681101d9d3a3 namespace: kserve-ci-e2e-test resourceVersion: "14658" uid: 077710e2-3f33-4f6f-b43d-407ad450a17f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:25:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-da412-predictor namespace: kserve-ci-e2e-test resourceVersion: "13933" uid: 4328ccc5-de01-47a1-aa7a-b23871698481 kind: Event lastTimestamp: "2026-04-24T21:27:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:25:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:27:05Z" name: isvc-sklearn-graph-raw-da412-predictor.18a96811023c9dd9 namespace: kserve-ci-e2e-test resourceVersion: "14659" uid: 72382c46-98ad-4b82-ac79-d16fd0f072aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:25:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-da412 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 1c8e95f7-be61-42da-8876-1770d15038e7 kind: Event lastTimestamp: "2026-04-24T21:26:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:26:32Z" name: isvc-sklearn-graph-raw-da412.18a9680a03b8f70f namespace: kserve-ci-e2e-test resourceVersion: "14377" uid: 641432ac-675c-4703-8c9d-bb325363e189 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-da412 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 1c8e95f7-be61-42da-8876-1770d15038e7 kind: Event lastTimestamp: "2026-04-24T21:25:20Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-da412": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-da412": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:25:20Z" name: isvc-sklearn-graph-raw-da412.18a9680a049f3983 namespace: kserve-ci-e2e-test resourceVersion: "13725" uid: 7fa672d3-e82d-4c11-9922-dee08ebfa367 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:26:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-da412 namespace: kserve-ci-e2e-test resourceVersion: "14375" uid: 1c8e95f7-be61-42da-8876-1770d15038e7 kind: Event lastTimestamp: "2026-04-24T21:26:32Z" message: InferenceService [isvc-sklearn-graph-raw-da412] is Ready metadata: creationTimestamp: "2026-04-24T21:26:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:26:32Z" name: isvc-sklearn-graph-raw-da412.18a9681ac6fef8e3 namespace: kserve-ci-e2e-test resourceVersion: "14376" uid: c0e37f28-2ebb-4621-8e9c-3592fec50821 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:27:10.553807Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 679512e3-c45f-4fa4-8854-212d65ef1285 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb to ip-10-0-132-118.ec2.internal metadata: creationTimestamp: "2026-04-24T21:27:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:27:10Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb.18a96823b592dceb namespace: kserve-ci-e2e-test resourceVersion: "14791" uid: d08ed934-8514-4e2e-a8c3-fa15e6b2633e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6b5fbb9956-pzrgr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb namespace: kserve-ci-e2e-test resourceVersion: "14808" uid: 679512e3-c45f-4fa4-8854-212d65ef1285 kind: Event lastTimestamp: "2026-04-24T21:27:10Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:27:10Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb.18a96823cebc1e49 namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: 8f58629c-c4b0-4956-a234-f3f2c64e63b7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 679512e3-c45f-4fa4-8854-212d65ef1285 kind: Event lastTimestamp: "2026-04-24T21:27:10Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:85f85c16414082de64de782f602228652c10c5bace08b518e803a7e911c62e5e" already present on machine metadata: creationTimestamp: "2026-04-24T21:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:10Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb.18a96823cff25bcb namespace: kserve-ci-e2e-test resourceVersion: "14832" uid: 5415c25d-41e1-4e67-9690-88d1d3cbc731 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 679512e3-c45f-4fa4-8854-212d65ef1285 kind: Event lastTimestamp: "2026-04-24T21:27:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:11Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb.18a96823d2c5384e namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: 300f5f47-70e7-4744-a4e3-a1a9e82ab028 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 679512e3-c45f-4fa4-8854-212d65ef1285 kind: Event lastTimestamp: "2026-04-24T21:27:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:11Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb.18a96823d333bc56 namespace: kserve-ci-e2e-test resourceVersion: "14835" uid: d191b072-823c-4053-9abe-2b354f8047fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 679512e3-c45f-4fa4-8854-212d65ef1285 kind: Event lastTimestamp: "2026-04-24T21:27:15Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:15Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb.18a96824d9d92d36 namespace: kserve-ci-e2e-test resourceVersion: "14897" uid: 61912e59-8e1d-45e5-b584-6f1b93d249bd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 679512e3-c45f-4fa4-8854-212d65ef1285 kind: Event lastTimestamp: "2026-04-24T21:27:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:15Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb.18a96824dd721e73 namespace: kserve-ci-e2e-test resourceVersion: "14903" uid: a6a8125d-2fec-4efa-ba72-ad582e5a761a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 679512e3-c45f-4fa4-8854-212d65ef1285 kind: Event lastTimestamp: "2026-04-24T21:27:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:15Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb.18a96824ddea9c73 namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: bc19d6bb-00da-4b90-99e8-9651d357a8e4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 679512e3-c45f-4fa4-8854-212d65ef1285 kind: Event lastTimestamp: "2026-04-24T21:27:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:15Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb.18a96824ddf735be namespace: kserve-ci-e2e-test resourceVersion: "14908" uid: 4fef1431-bc62-486c-8751-a569a284f79c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 679512e3-c45f-4fa4-8854-212d65ef1285 kind: Event lastTimestamp: "2026-04-24T21:27:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:15Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb.18a96824e131dca9 namespace: kserve-ci-e2e-test resourceVersion: "14915" uid: e145d9fd-4e84-4df6-a3cf-1b2221dd2bde reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 679512e3-c45f-4fa4-8854-212d65ef1285 kind: Event lastTimestamp: "2026-04-24T21:27:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:15Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb.18a96824e1ac56e7 namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: 7b5a9ece-a5b3-4137-a8b9-2a0ee0229e4d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:27:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 679512e3-c45f-4fa4-8854-212d65ef1285 kind: Event lastTimestamp: "2026-04-24T21:28:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:53Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb.18a9682551f1a267 namespace: kserve-ci-e2e-test resourceVersion: "15813" uid: 4b1bdd1c-c352-4331-8e6c-95d70aad21db reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 679512e3-c45f-4fa4-8854-212d65ef1285 kind: Event lastTimestamp: "2026-04-24T21:28:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:50Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb.18a9683b092fb4d0 namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: ad156189-5c72-4127-a220-49b304b8459b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 679512e3-c45f-4fa4-8854-212d65ef1285 kind: Event lastTimestamp: "2026-04-24T21:28:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:50Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb.18a9683b09302790 namespace: kserve-ci-e2e-test resourceVersion: "15742" uid: f0f9fd8a-c478-4104-bf19-5961aed42bd1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 679512e3-c45f-4fa4-8854-212d65ef1285 kind: Event lastTimestamp: "2026-04-24T21:28:53Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:53Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb.18a9683bac1b1ecf namespace: kserve-ci-e2e-test resourceVersion: "15812" uid: e5facd6c-71f4-4d7d-8192-71ecc2dcbb43 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456 namespace: kserve-ci-e2e-test resourceVersion: "14760" uid: 2f03edc7-1498-45af-b118-106fa1778609 kind: Event lastTimestamp: "2026-04-24T21:27:10Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456-dphqb' metadata: creationTimestamp: "2026-04-24T21:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:27:10Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456.18a96823b53f6733 namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: bbad31a8-77d9-4887-ad02-7c145af0bd68 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-07ea4-predictor namespace: kserve-ci-e2e-test resourceVersion: "14758" uid: 8efffea7-5511-48b9-884f-1e4782a7c02f kind: Event lastTimestamp: "2026-04-24T21:27:10Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-07ea4-predictor-6c64cf7456 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:27:10Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor.18a96823b0f8d921 namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: e1699345-3189-4755-a0ad-6708a81af0a2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:27:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-07ea4-predictor namespace: kserve-ci-e2e-test resourceVersion: "14759" uid: 7e901e7c-83a3-4f54-9c1a-def427b00eab kind: Event lastTimestamp: "2026-04-24T21:27:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:27:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:27:55Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor.18a968272fa2f1de namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: 6ead77d9-3e4f-446c-b0d4-61f678a733c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:27:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-07ea4-predictor namespace: kserve-ci-e2e-test resourceVersion: "14759" uid: 7e901e7c-83a3-4f54-9c1a-def427b00eab kind: Event lastTimestamp: "2026-04-24T21:27:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:27:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:27:55Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor.18a96827300efcb4 namespace: kserve-ci-e2e-test resourceVersion: "15230" uid: 53c8f83d-6ff2-4244-8c35-995d2af36d1a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:28:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-07ea4-predictor namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: 7e901e7c-83a3-4f54-9c1a-def427b00eab kind: Event lastTimestamp: "2026-04-24T21:28:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:28:40Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor.18a96831ad244797 namespace: kserve-ci-e2e-test resourceVersion: "15595" uid: 3dc0681c-75dd-4bd3-b7d0-e9e0452f6bf2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:28:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-07ea4-predictor namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: 7e901e7c-83a3-4f54-9c1a-def427b00eab kind: Event lastTimestamp: "2026-04-24T21:28:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:28:40Z" name: isvc-sklearn-graph-raw-hpa-07ea4-predictor.18a96831ad848d55 namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: 670bfeb5-5eb6-4a3a-896b-557afa27d1c8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:27:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-07ea4 namespace: kserve-ci-e2e-test resourceVersion: "14747" uid: cdbb6c68-f632-4986-a329-36baf04cce88 kind: Event lastTimestamp: "2026-04-24T21:28:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:28:23Z" name: isvc-sklearn-graph-raw-hpa-07ea4.18a96823b0eac9c4 namespace: kserve-ci-e2e-test resourceVersion: "15449" uid: 7184f203-bbf6-4761-b8d1-6a0751595347 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-07ea4 namespace: kserve-ci-e2e-test resourceVersion: "15446" uid: cdbb6c68-f632-4986-a329-36baf04cce88 kind: Event lastTimestamp: "2026-04-24T21:28:23Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-07ea4] is Ready metadata: creationTimestamp: "2026-04-24T21:28:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:28:23Z" name: isvc-sklearn-graph-raw-hpa-07ea4.18a96834b5683986 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 6d3cdf82-b025-405f-9a19-15721351e584 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:30:36.082191Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg namespace: kserve-ci-e2e-test resourceVersion: "16693" uid: 6112d18d-543e-4963-a027-c640879c9c06 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg to ip-10-0-132-118.ec2.internal metadata: creationTimestamp: "2026-04-24T21:30:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:30:36Z" name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg.18a9685390051650 namespace: kserve-ci-e2e-test resourceVersion: "16701" uid: 201d93cd-c686-408f-b0b6-fd61f47cce98 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6b5fbb9956-pzrgr source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:30:36Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 6112d18d-543e-4963-a027-c640879c9c06 kind: Event lastTimestamp: "2026-04-24T21:30:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-63746-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:30:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:36Z" name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg.18a968539cb993be namespace: kserve-ci-e2e-test resourceVersion: "16731" uid: c9f5ea71-bb73-45e0-b44b-a07f6edabc5f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:37Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg namespace: kserve-ci-e2e-test resourceVersion: "16711" uid: 6112d18d-543e-4963-a027-c640879c9c06 kind: Event lastTimestamp: "2026-04-24T21:30:37Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:30:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:30:37Z" name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg.18a9685402313799 namespace: kserve-ci-e2e-test resourceVersion: "16743" uid: d30b8d8d-8f6e-4f81-846a-30e3edd349f9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 6112d18d-543e-4963-a027-c640879c9c06 kind: Event lastTimestamp: "2026-04-24T21:30:38Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:85f85c16414082de64de782f602228652c10c5bace08b518e803a7e911c62e5e" already present on machine metadata: creationTimestamp: "2026-04-24T21:30:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:38Z" name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg.18a9685403c6b24e namespace: kserve-ci-e2e-test resourceVersion: "16746" uid: 725f8f33-7134-4034-8008-36b54a7b63aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 6112d18d-543e-4963-a027-c640879c9c06 kind: Event lastTimestamp: "2026-04-24T21:30:38Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:30:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:38Z" name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg.18a9685406e55488 namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 1338a55f-aa67-422f-9149-9a0c3fd0e417 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 6112d18d-543e-4963-a027-c640879c9c06 kind: Event lastTimestamp: "2026-04-24T21:30:38Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:30:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:38Z" name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg.18a9685407531ce7 namespace: kserve-ci-e2e-test resourceVersion: "16749" uid: d6cc6212-18a1-4270-b00e-25762c63052e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 6112d18d-543e-4963-a027-c640879c9c06 kind: Event lastTimestamp: "2026-04-24T21:30:42Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:42Z" name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg.18a96854f7f5a480 namespace: kserve-ci-e2e-test resourceVersion: "16784" uid: 54871ab5-98da-43d0-be52-e8504dec21f0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 6112d18d-543e-4963-a027-c640879c9c06 kind: Event lastTimestamp: "2026-04-24T21:30:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:42Z" name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg.18a96854facffa13 namespace: kserve-ci-e2e-test resourceVersion: "16786" uid: e0418f02-534a-4ffe-944a-80145c2fbe27 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 6112d18d-543e-4963-a027-c640879c9c06 kind: Event lastTimestamp: "2026-04-24T21:30:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:42Z" name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg.18a96854fb417e9f namespace: kserve-ci-e2e-test resourceVersion: "16789" uid: c1050d58-4d2a-4ba4-8a17-957a6fbdf986 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 6112d18d-543e-4963-a027-c640879c9c06 kind: Event lastTimestamp: "2026-04-24T21:30:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:42Z" name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg.18a96854fb4da749 namespace: kserve-ci-e2e-test resourceVersion: "16790" uid: c01d78c9-1ca8-47d4-aa9a-7a2b1d102537 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 6112d18d-543e-4963-a027-c640879c9c06 kind: Event lastTimestamp: "2026-04-24T21:30:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:42Z" name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg.18a96854fe562577 namespace: kserve-ci-e2e-test resourceVersion: "16793" uid: 2181ea1e-0825-428e-b808-86daae4dcd32 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 6112d18d-543e-4963-a027-c640879c9c06 kind: Event lastTimestamp: "2026-04-24T21:30:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:42Z" name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg.18a96854fec0ece6 namespace: kserve-ci-e2e-test resourceVersion: "16794" uid: 92376ec1-0414-454e-9cf1-16d3ffd85545 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 16 eventTime: null firstTimestamp: "2026-04-24T21:30:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 6112d18d-543e-4963-a027-c640879c9c06 kind: Event lastTimestamp: "2026-04-24T21:33:06Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:30:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:33:06Z" name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg.18a968556fa94496 namespace: kserve-ci-e2e-test resourceVersion: "17884" uid: 0d313f3f-8748-4f7d-8e87-f03fc12a404a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:33:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 6112d18d-543e-4963-a027-c640879c9c06 kind: Event lastTimestamp: "2026-04-24T21:33:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:33:06Z" name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg.18a968768641ab8f namespace: kserve-ci-e2e-test resourceVersion: "17856" uid: 05dee8cf-5ecd-46a2-9988-761c53153180 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:33:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 6112d18d-543e-4963-a027-c640879c9c06 kind: Event lastTimestamp: "2026-04-24T21:33:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:33:06Z" name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg.18a9687686424340 namespace: kserve-ci-e2e-test resourceVersion: "17859" uid: 9cbbf64a-4241-41e6-99d1-8cc7c2c0f4db reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:33:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 6112d18d-543e-4963-a027-c640879c9c06 kind: Event lastTimestamp: "2026-04-24T21:33:10Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:33:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:33:10Z" name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg.18a968776e21fd29 namespace: kserve-ci-e2e-test resourceVersion: "17925" uid: 176d4123-5e57-4e88-85bb-b26ae83c35c8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b namespace: kserve-ci-e2e-test resourceVersion: "16674" uid: 76a52bc8-afac-4595-afb3-57fc5f0c1285 kind: Event lastTimestamp: "2026-04-24T21:30:36Z" message: 'Created pod: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b-z87xg' metadata: creationTimestamp: "2026-04-24T21:30:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:36Z" name: isvc-sklearn-scale-raw-63746-predictor-5769d66c6b.18a968538f630830 namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: 18ee7fc2-fcff-4c81-bcfd-3a5b99bfb606 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-63746-predictor namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: 5e46626f-aebc-40b3-926d-47072e0ee78f kind: Event lastTimestamp: "2026-04-24T21:30:36Z" message: Scaled up replica set isvc-sklearn-scale-raw-63746-predictor-5769d66c6b from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:30:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:36Z" name: isvc-sklearn-scale-raw-63746-predictor.18a968538c097651 namespace: kserve-ci-e2e-test resourceVersion: "16679" uid: 5e254e4e-82f0-43cb-a9f0-209f556ffa1a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:30:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-63746-predictor namespace: kserve-ci-e2e-test resourceVersion: "16673" uid: e6215809-af86-4f33-8e82-883fb8838587 kind: Event lastTimestamp: "2026-04-24T21:31:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:31:21Z" name: isvc-sklearn-scale-raw-63746-predictor.18a968570b1a2259 namespace: kserve-ci-e2e-test resourceVersion: "17081" uid: 8741d0f3-6401-4b0f-9c70-69bbd7ac4389 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:30:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-63746-predictor namespace: kserve-ci-e2e-test resourceVersion: "16673" uid: e6215809-af86-4f33-8e82-883fb8838587 kind: Event lastTimestamp: "2026-04-24T21:31:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:31:21Z" name: isvc-sklearn-scale-raw-63746-predictor.18a968570b88fbff namespace: kserve-ci-e2e-test resourceVersion: "17082" uid: a0267151-c62b-4dff-a642-20a920836bd2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T21:31:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-63746-predictor namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: e6215809-af86-4f33-8e82-883fb8838587 kind: Event lastTimestamp: "2026-04-24T21:33:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:31:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:33:06Z" name: isvc-sklearn-scale-raw-63746-predictor.18a9685a8a4165d8 namespace: kserve-ci-e2e-test resourceVersion: "17834" uid: 82eae665-584e-4760-994b-2ac01a905265 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T21:31:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-63746-predictor namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: e6215809-af86-4f33-8e82-883fb8838587 kind: Event lastTimestamp: "2026-04-24T21:33:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:31:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:33:06Z" name: isvc-sklearn-scale-raw-63746-predictor.18a9685a8aacb857 namespace: kserve-ci-e2e-test resourceVersion: "17835" uid: 4dd0b693-27e1-41ad-a893-9f931e5a2bb6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T21:30:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-63746 namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: e960b8df-fb6a-4097-ab69-7d44347f4202 kind: Event lastTimestamp: "2026-04-24T21:32:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:30:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:32:56Z" name: isvc-sklearn-scale-raw-63746.18a968538c27f709 namespace: kserve-ci-e2e-test resourceVersion: "17755" uid: 7846c1e7-b6fe-48c8-b89a-7a016e7d467c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-63746 namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: e960b8df-fb6a-4097-ab69-7d44347f4202 kind: Event lastTimestamp: "2026-04-24T21:30:36Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-63746": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-63746": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:30:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:30:36Z" name: isvc-sklearn-scale-raw-63746.18a968538cb03f9d namespace: kserve-ci-e2e-test resourceVersion: "16686" uid: 76cfc2cc-f96b-47bd-88a6-1b2c7ee53f51 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:32:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-63746 namespace: kserve-ci-e2e-test resourceVersion: "17753" uid: e960b8df-fb6a-4097-ab69-7d44347f4202 kind: Event lastTimestamp: "2026-04-24T21:32:56Z" message: InferenceService [isvc-sklearn-scale-raw-63746] is Ready metadata: creationTimestamp: "2026-04-24T21:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:32:56Z" name: isvc-sklearn-scale-raw-63746.18a9687443a91758 namespace: kserve-ci-e2e-test resourceVersion: "17754" uid: 26c6379d-40e1-482f-9221-7080a12a1c40 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:25:20.415583Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl namespace: kserve-ci-e2e-test resourceVersion: "13761" uid: a347c45e-1f6f-4252-a817-9390a6dd3ec5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl to ip-10-0-132-118.ec2.internal metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:25:20Z" name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl.18a9680a10d2ea74 namespace: kserve-ci-e2e-test resourceVersion: "13768" uid: 4cd3731d-2d43-4e54-a4ad-1d9a842214a7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6b5fbb9956-pzrgr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl namespace: kserve-ci-e2e-test resourceVersion: "13774" uid: a347c45e-1f6f-4252-a817-9390a6dd3ec5 kind: Event lastTimestamp: "2026-04-24T21:25:20Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:25:20Z" name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl.18a9680a2aa4dac8 namespace: kserve-ci-e2e-test resourceVersion: "13786" uid: dd4e0dc8-e0c2-43d6-b691-1808ce59b250 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl namespace: kserve-ci-e2e-test resourceVersion: "13765" uid: a347c45e-1f6f-4252-a817-9390a6dd3ec5 kind: Event lastTimestamp: "2026-04-24T21:25:20Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:85f85c16414082de64de782f602228652c10c5bace08b518e803a7e911c62e5e" already present on machine metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:20Z" name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl.18a9680a2c38be2c namespace: kserve-ci-e2e-test resourceVersion: "13788" uid: c20c761b-a17d-460e-8bb5-142925f1aea4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl namespace: kserve-ci-e2e-test resourceVersion: "13765" uid: a347c45e-1f6f-4252-a817-9390a6dd3ec5 kind: Event lastTimestamp: "2026-04-24T21:25:20Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:20Z" name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl.18a9680a2f12757a namespace: kserve-ci-e2e-test resourceVersion: "13790" uid: 6c5adbd0-d162-458f-b571-d17d548a0b0d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl namespace: kserve-ci-e2e-test resourceVersion: "13765" uid: a347c45e-1f6f-4252-a817-9390a6dd3ec5 kind: Event lastTimestamp: "2026-04-24T21:25:20Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:20Z" name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl.18a9680a2f81b102 namespace: kserve-ci-e2e-test resourceVersion: "13791" uid: 5de2064e-67dd-41b6-ae16-77858dabc45d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl namespace: kserve-ci-e2e-test resourceVersion: "13765" uid: a347c45e-1f6f-4252-a817-9390a6dd3ec5 kind: Event lastTimestamp: "2026-04-24T21:25:25Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-24T21:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:25Z" name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl.18a9680b278f6156 namespace: kserve-ci-e2e-test resourceVersion: "13843" uid: 19733aa3-3841-4a5d-b2ac-7350a9578e26 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl namespace: kserve-ci-e2e-test resourceVersion: "13765" uid: a347c45e-1f6f-4252-a817-9390a6dd3ec5 kind: Event lastTimestamp: "2026-04-24T21:25:43Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 18.745s (18.745s including waiting). Image size: 1306417402 bytes.' metadata: creationTimestamp: "2026-04-24T21:25:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:43Z" name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl.18a9680f84dc0d76 namespace: kserve-ci-e2e-test resourceVersion: "13996" uid: 731a5f47-a672-4ce9-8cff-a8319777a9e2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl namespace: kserve-ci-e2e-test resourceVersion: "13765" uid: a347c45e-1f6f-4252-a817-9390a6dd3ec5 kind: Event lastTimestamp: "2026-04-24T21:25:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:25:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:43Z" name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl.18a9680f88376dd9 namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: da52ec98-3c05-4be7-8c0e-6acb08aaa629 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl namespace: kserve-ci-e2e-test resourceVersion: "13765" uid: a347c45e-1f6f-4252-a817-9390a6dd3ec5 kind: Event lastTimestamp: "2026-04-24T21:25:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:25:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:43Z" name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl.18a9680f88a55c7d namespace: kserve-ci-e2e-test resourceVersion: "14000" uid: 82202d80-4fe7-4f68-84ef-ea8aabc4358a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl namespace: kserve-ci-e2e-test resourceVersion: "13765" uid: a347c45e-1f6f-4252-a817-9390a6dd3ec5 kind: Event lastTimestamp: "2026-04-24T21:25:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:25:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:43Z" name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl.18a9680f88b24d38 namespace: kserve-ci-e2e-test resourceVersion: "14001" uid: 8eae9e96-8e19-4681-b990-48de1ed8bbd5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl namespace: kserve-ci-e2e-test resourceVersion: "13765" uid: a347c45e-1f6f-4252-a817-9390a6dd3ec5 kind: Event lastTimestamp: "2026-04-24T21:25:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:25:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:43Z" name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl.18a9680f8b7bed11 namespace: kserve-ci-e2e-test resourceVersion: "14003" uid: 236df866-a4f8-4633-9205-46cffc2655d7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl namespace: kserve-ci-e2e-test resourceVersion: "13765" uid: a347c45e-1f6f-4252-a817-9390a6dd3ec5 kind: Event lastTimestamp: "2026-04-24T21:25:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:25:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:25:43Z" name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl.18a9680f8bedda9d namespace: kserve-ci-e2e-test resourceVersion: "14004" uid: a962ffd2-fa64-4ed8-878d-0b44bf0bddef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:25:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl namespace: kserve-ci-e2e-test resourceVersion: "13765" uid: a347c45e-1f6f-4252-a817-9390a6dd3ec5 kind: Event lastTimestamp: "2026-04-24T21:27:11Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:25:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:11Z" name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl.18a9680fd44b3053 namespace: kserve-ci-e2e-test resourceVersion: "14840" uid: 3f89c841-ed69-4136-9d95-cfbfd4692fb2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl namespace: kserve-ci-e2e-test resourceVersion: "13765" uid: a347c45e-1f6f-4252-a817-9390a6dd3ec5 kind: Event lastTimestamp: "2026-04-24T21:27:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:10Z" name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl.18a96823b5a453a9 namespace: kserve-ci-e2e-test resourceVersion: "14794" uid: f31c6c17-8865-4633-b854-80b0e864e3af reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl namespace: kserve-ci-e2e-test resourceVersion: "13765" uid: a347c45e-1f6f-4252-a817-9390a6dd3ec5 kind: Event lastTimestamp: "2026-04-24T21:27:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:10Z" name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl.18a96823b5a7035b namespace: kserve-ci-e2e-test resourceVersion: "14798" uid: e82d49d5-a22d-4ac0-9da8-a5d86f8f0638 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl namespace: kserve-ci-e2e-test resourceVersion: "13765" uid: a347c45e-1f6f-4252-a817-9390a6dd3ec5 kind: Event lastTimestamp: "2026-04-24T21:27:11Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:11Z" name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl.18a96823da68806f namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: 6955b1f7-2f28-4f76-a358-e254e8e9eaf7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7 namespace: kserve-ci-e2e-test resourceVersion: "13747" uid: 04a6727d-02bc-4c77-a9e7-0d0a0ba0c6dd kind: Event lastTimestamp: "2026-04-24T21:25:20Z" message: 'Created pod: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7-xdnhl' metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:25:20Z" name: isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7.18a9680a102d234c namespace: kserve-ci-e2e-test resourceVersion: "13763" uid: d0777247-d4e6-43a8-8577-37f703fc3058 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:25:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-da412-predictor namespace: kserve-ci-e2e-test resourceVersion: "13742" uid: f01afbf6-c23f-4510-aa01-213d34d4c4d0 kind: Event lastTimestamp: "2026-04-24T21:25:20Z" message: Scaled up replica set isvc-xgboost-graph-raw-da412-predictor-9b7c6cff7 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:25:20Z" name: isvc-xgboost-graph-raw-da412-predictor.18a9680a0ca074df namespace: kserve-ci-e2e-test resourceVersion: "13753" uid: d3dbd205-cf93-43f0-a3db-b1b0507fcf6c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:25:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-da412-predictor namespace: kserve-ci-e2e-test resourceVersion: "13746" uid: 3de28267-7c3b-48ad-b41f-464ad5925567 kind: Event lastTimestamp: "2026-04-24T21:25:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:25:50Z" name: isvc-xgboost-graph-raw-da412-predictor.18a9680d8b427130 namespace: kserve-ci-e2e-test resourceVersion: "14058" uid: 7613bf76-1bc6-4a6a-bee1-928322a00d3f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:25:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-da412-predictor namespace: kserve-ci-e2e-test resourceVersion: "13746" uid: 3de28267-7c3b-48ad-b41f-464ad5925567 kind: Event lastTimestamp: "2026-04-24T21:25:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:25:50Z" name: isvc-xgboost-graph-raw-da412-predictor.18a9680d8bb8e677 namespace: kserve-ci-e2e-test resourceVersion: "14059" uid: b6c2e630-3a69-485d-a2e6-285463ca7eb4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:26:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-da412-predictor namespace: kserve-ci-e2e-test resourceVersion: "13937" uid: 3de28267-7c3b-48ad-b41f-464ad5925567 kind: Event lastTimestamp: "2026-04-24T21:27:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:27:05Z" name: isvc-xgboost-graph-raw-da412-predictor.18a9681489118d0e namespace: kserve-ci-e2e-test resourceVersion: "14661" uid: f41bcdad-0614-4da1-bad2-c24e91f08b5e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:26:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-da412-predictor namespace: kserve-ci-e2e-test resourceVersion: "13937" uid: 3de28267-7c3b-48ad-b41f-464ad5925567 kind: Event lastTimestamp: "2026-04-24T21:27:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:27:05Z" name: isvc-xgboost-graph-raw-da412-predictor.18a9681489743263 namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 20307988-1c58-46a4-99a0-f33ee7c34d70 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T21:25:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-da412 namespace: kserve-ci-e2e-test resourceVersion: "13732" uid: 252a347c-4122-413b-b1ee-c07cdd2b0835 kind: Event lastTimestamp: "2026-04-24T21:26:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:26:51Z" name: isvc-xgboost-graph-raw-da412.18a9680a0c8c77d9 namespace: kserve-ci-e2e-test resourceVersion: "14523" uid: f510239c-a984-4348-af87-0acca5522d70 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:26:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-da412 namespace: kserve-ci-e2e-test resourceVersion: "14520" uid: 252a347c-4122-413b-b1ee-c07cdd2b0835 kind: Event lastTimestamp: "2026-04-24T21:26:51Z" message: InferenceService [isvc-xgboost-graph-raw-da412] is Ready metadata: creationTimestamp: "2026-04-24T21:26:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:26:51Z" name: isvc-xgboost-graph-raw-da412.18a9681f37aa4e7e namespace: kserve-ci-e2e-test resourceVersion: "14521" uid: 711b6d30-acce-4112-a89a-2012aad12ddb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:27:10.655581Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: 7f8f1183-7c95-4acf-9967-2c2db7cbe81d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf to ip-10-0-132-118.ec2.internal metadata: creationTimestamp: "2026-04-24T21:27:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:27:10Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf.18a96823bba3df86 namespace: kserve-ci-e2e-test resourceVersion: "14820" uid: 59daf361-2db7-4deb-a1d1-5624ba99ff6e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6b5fbb9956-pzrgr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 7f8f1183-7c95-4acf-9967-2c2db7cbe81d kind: Event lastTimestamp: "2026-04-24T21:27:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-hpa-07ea4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:10Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf.18a96823c4b03516 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: 4342037c-5efe-49b4-83e4-750e508649d5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf namespace: kserve-ci-e2e-test resourceVersion: "14824" uid: 7f8f1183-7c95-4acf-9967-2c2db7cbe81d kind: Event lastTimestamp: "2026-04-24T21:27:11Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:27:11Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf.18a96823f8799973 namespace: kserve-ci-e2e-test resourceVersion: "14850" uid: 1212fcc2-bdf3-45eb-b88e-f4bfdf8e7d76 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 7f8f1183-7c95-4acf-9967-2c2db7cbe81d kind: Event lastTimestamp: "2026-04-24T21:27:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:85f85c16414082de64de782f602228652c10c5bace08b518e803a7e911c62e5e" already present on machine metadata: creationTimestamp: "2026-04-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:11Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf.18a96823f9dc0699 namespace: kserve-ci-e2e-test resourceVersion: "14852" uid: 2a9db583-4ba6-4889-aabc-e0993d5ca415 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 7f8f1183-7c95-4acf-9967-2c2db7cbe81d kind: Event lastTimestamp: "2026-04-24T21:27:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:11Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf.18a96823fca35a00 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: deb8fd54-b8a1-4130-8782-37ac4eb4aeab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 7f8f1183-7c95-4acf-9967-2c2db7cbe81d kind: Event lastTimestamp: "2026-04-24T21:27:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:11Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf.18a96823fd0d9069 namespace: kserve-ci-e2e-test resourceVersion: "14854" uid: fe5a3bc5-d893-46b6-bcb8-44c6c911589e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 7f8f1183-7c95-4acf-9967-2c2db7cbe81d kind: Event lastTimestamp: "2026-04-24T21:27:15Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-24T21:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:15Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf.18a96824da156303 namespace: kserve-ci-e2e-test resourceVersion: "14898" uid: f8e42f82-c426-4855-9be4-cdbbfac46396 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 7f8f1183-7c95-4acf-9967-2c2db7cbe81d kind: Event lastTimestamp: "2026-04-24T21:27:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:15Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf.18a96824dd7e0f07 namespace: kserve-ci-e2e-test resourceVersion: "14904" uid: 45b0e7ad-d5bb-49cc-8f18-89a33e33c9fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 7f8f1183-7c95-4acf-9967-2c2db7cbe81d kind: Event lastTimestamp: "2026-04-24T21:27:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:15Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf.18a96824ddf8e4ec namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: f00e21af-1a81-4ab7-8b70-93c2967dd69b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 7f8f1183-7c95-4acf-9967-2c2db7cbe81d kind: Event lastTimestamp: "2026-04-24T21:27:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:15Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf.18a96824de049c88 namespace: kserve-ci-e2e-test resourceVersion: "14912" uid: 429b96d6-2b47-458d-8d77-167b0247588e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 7f8f1183-7c95-4acf-9967-2c2db7cbe81d kind: Event lastTimestamp: "2026-04-24T21:27:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:15Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf.18a96824e148001e namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: d5eb73ee-fb2f-4192-92b8-6665482a851d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 7f8f1183-7c95-4acf-9967-2c2db7cbe81d kind: Event lastTimestamp: "2026-04-24T21:27:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:15Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf.18a96824e1d002f8 namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: 4e464958-5eba-4244-9a12-7339790575b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T21:27:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 7f8f1183-7c95-4acf-9967-2c2db7cbe81d kind: Event lastTimestamp: "2026-04-24T21:28:52Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:27:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:52Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf.18a96825162804ce namespace: kserve-ci-e2e-test resourceVersion: "15798" uid: 7c1ff3f5-63fe-43d0-85ec-b70d4848bcce reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 7f8f1183-7c95-4acf-9967-2c2db7cbe81d kind: Event lastTimestamp: "2026-04-24T21:28:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:50Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf.18a9683b0debac97 namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: 5928adee-bbe4-48a1-868c-bd3ea2247836 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 7f8f1183-7c95-4acf-9967-2c2db7cbe81d kind: Event lastTimestamp: "2026-04-24T21:28:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:50Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf.18a9683b0dec20b8 namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: 75fb502c-dd59-477c-8f50-168f3ff666cd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 7f8f1183-7c95-4acf-9967-2c2db7cbe81d kind: Event lastTimestamp: "2026-04-24T21:28:52Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:28:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:52Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf.18a9683b7058e4a0 namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: 8ed9e2ca-781a-4c04-a42b-0a484a96d6d7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77 namespace: kserve-ci-e2e-test resourceVersion: "14802" uid: 773d484e-85f6-45dd-9bb9-503c8a70cbf4 kind: Event lastTimestamp: "2026-04-24T21:27:10Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77-2nfpf' metadata: creationTimestamp: "2026-04-24T21:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:27:10Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77.18a96823bb42a6cf namespace: kserve-ci-e2e-test resourceVersion: "14817" uid: fc5ac334-2925-43c2-835e-68760a6759f4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-07ea4-predictor namespace: kserve-ci-e2e-test resourceVersion: "14796" uid: 86af2c17-b46c-4c8a-b7d9-5e2a35e07435 kind: Event lastTimestamp: "2026-04-24T21:27:10Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-07ea4-predictor-677dc57f77 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:27:10Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor.18a96823b7470443 namespace: kserve-ci-e2e-test resourceVersion: "14804" uid: 902be7b6-9cda-40e0-b2a9-686773114c45 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:27:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-07ea4-predictor namespace: kserve-ci-e2e-test resourceVersion: "14799" uid: 1fc2fa09-9852-467e-b84c-a8cc40c3f361 kind: Event lastTimestamp: "2026-04-24T21:27:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:27:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:27:55Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor.18a9682735c30e67 namespace: kserve-ci-e2e-test resourceVersion: "15232" uid: 75719d4e-f957-4eaa-a5b3-517cdfbff44d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:27:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-07ea4-predictor namespace: kserve-ci-e2e-test resourceVersion: "14799" uid: 1fc2fa09-9852-467e-b84c-a8cc40c3f361 kind: Event lastTimestamp: "2026-04-24T21:27:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:27:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:27:55Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor.18a9682736298170 namespace: kserve-ci-e2e-test resourceVersion: "15234" uid: 44323658-5e4e-4947-a38c-e03df8c83568 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:27:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-07ea4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 1fc2fa09-9852-467e-b84c-a8cc40c3f361 kind: Event lastTimestamp: "2026-04-24T21:28:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:28:40Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor.18a9682ab4d157f6 namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: fb70f0bf-a788-45ad-b5ae-a7150afe96fd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:27:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-07ea4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 1fc2fa09-9852-467e-b84c-a8cc40c3f361 kind: Event lastTimestamp: "2026-04-24T21:28:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:28:40Z" name: isvc-xgboost-graph-raw-hpa-07ea4-predictor.18a9682ab5317df1 namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: 4da5ce4b-cefb-40f7-833a-1ff470e7984c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T21:27:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-07ea4 namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 52711828-7cd4-44a6-8dfc-5a885ee6efeb kind: Event lastTimestamp: "2026-04-24T21:28:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:28:12Z" name: isvc-xgboost-graph-raw-hpa-07ea4.18a96823b72bd9bf namespace: kserve-ci-e2e-test resourceVersion: "15363" uid: 6aeee8d6-52d9-4817-b126-be897af4cc9c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-07ea4 namespace: kserve-ci-e2e-test resourceVersion: "15361" uid: 52711828-7cd4-44a6-8dfc-5a885ee6efeb kind: Event lastTimestamp: "2026-04-24T21:28:12Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-07ea4] is Ready metadata: creationTimestamp: "2026-04-24T21:28:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:28:12Z" name: isvc-xgboost-graph-raw-hpa-07ea4.18a9683225625318 namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: 9d73f5cf-d31b-400d-a2be-015968ef87e1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:28:50.777446Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-66937-predictor-86db465969-zxjgs namespace: kserve-ci-e2e-test resourceVersion: "15744" uid: cf53e686-46b5-470b-90cf-27b7cba22991 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-66937-predictor-86db465969-zxjgs to ip-10-0-132-118.ec2.internal metadata: creationTimestamp: "2026-04-24T21:28:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:28:50Z" name: message-dumper-raw-66937-predictor-86db465969-zxjgs.18a9683b0b5e3d26 namespace: kserve-ci-e2e-test resourceVersion: "15751" uid: ff0adeb5-13b3-483f-83fb-f939c7647e29 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6b5fbb9956-pzrgr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:51Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-66937-predictor-86db465969-zxjgs namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: cf53e686-46b5-470b-90cf-27b7cba22991 kind: Event lastTimestamp: "2026-04-24T21:28:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-raw-66937-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:28:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:51Z" name: message-dumper-raw-66937-predictor-86db465969-zxjgs.18a9683b1a14f2f3 namespace: kserve-ci-e2e-test resourceVersion: "15776" uid: e6e54d00-6ae5-4fe4-b6b8-3e2d2a760aa0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:51Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-66937-predictor-86db465969-zxjgs namespace: kserve-ci-e2e-test resourceVersion: "15765" uid: cf53e686-46b5-470b-90cf-27b7cba22991 kind: Event lastTimestamp: "2026-04-24T21:28:51Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:28:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:28:51Z" name: message-dumper-raw-66937-predictor-86db465969-zxjgs.18a9683b481b8f56 namespace: kserve-ci-e2e-test resourceVersion: "15787" uid: f27459b3-1573-4fa7-849f-fc0e826f4d31 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-66937-predictor-86db465969-zxjgs namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: cf53e686-46b5-470b-90cf-27b7cba22991 kind: Event lastTimestamp: "2026-04-24T21:28:51Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-24T21:28:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:51Z" name: message-dumper-raw-66937-predictor-86db465969-zxjgs.18a9683b49686a99 namespace: kserve-ci-e2e-test resourceVersion: "15789" uid: 14ff4af8-04c2-40a7-ae5e-f1c49237deda reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-66937-predictor-86db465969-zxjgs namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: cf53e686-46b5-470b-90cf-27b7cba22991 kind: Event lastTimestamp: "2026-04-24T21:28:53Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.317s (1.317s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-24T21:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:53Z" name: message-dumper-raw-66937-predictor-86db465969-zxjgs.18a9683b97f592f9 namespace: kserve-ci-e2e-test resourceVersion: "15802" uid: 45aaf296-8079-4747-b169-848c9462636f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-66937-predictor-86db465969-zxjgs namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: cf53e686-46b5-470b-90cf-27b7cba22991 kind: Event lastTimestamp: "2026-04-24T21:28:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:53Z" name: message-dumper-raw-66937-predictor-86db465969-zxjgs.18a9683b9b1b1b4f namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: a5943f0f-8e5d-46c7-bd62-29af70308321 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-66937-predictor-86db465969-zxjgs namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: cf53e686-46b5-470b-90cf-27b7cba22991 kind: Event lastTimestamp: "2026-04-24T21:28:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:53Z" name: message-dumper-raw-66937-predictor-86db465969-zxjgs.18a9683b9b86c263 namespace: kserve-ci-e2e-test resourceVersion: "15806" uid: 9443a98c-e6d1-492b-9aec-ed2e52b3bcf4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-66937-predictor-86db465969-zxjgs namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: cf53e686-46b5-470b-90cf-27b7cba22991 kind: Event lastTimestamp: "2026-04-24T21:28:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:53Z" name: message-dumper-raw-66937-predictor-86db465969-zxjgs.18a9683b9b91364d namespace: kserve-ci-e2e-test resourceVersion: "15807" uid: 3ebcb54f-843b-4d4e-bb5b-d586ff502fc3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-66937-predictor-86db465969-zxjgs namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: cf53e686-46b5-470b-90cf-27b7cba22991 kind: Event lastTimestamp: "2026-04-24T21:28:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:53Z" name: message-dumper-raw-66937-predictor-86db465969-zxjgs.18a9683b9e543440 namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: 3063aa58-8318-4ad4-bc90-49c6bf333283 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-66937-predictor-86db465969-zxjgs namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: cf53e686-46b5-470b-90cf-27b7cba22991 kind: Event lastTimestamp: "2026-04-24T21:28:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:53Z" name: message-dumper-raw-66937-predictor-86db465969-zxjgs.18a9683b9ec4a02b namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: 4556f068-3b5f-4bc5-90a7-6af6ee78a810 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-66937-predictor-86db465969-zxjgs namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: cf53e686-46b5-470b-90cf-27b7cba22991 kind: Event lastTimestamp: "2026-04-24T21:30:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:30:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:36Z" name: message-dumper-raw-66937-predictor-86db465969-zxjgs.18a9685392ba38f2 namespace: kserve-ci-e2e-test resourceVersion: "16718" uid: 5857e81b-fc03-4bae-a0fe-4e46f329b630 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-66937-predictor-86db465969-zxjgs namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: cf53e686-46b5-470b-90cf-27b7cba22991 kind: Event lastTimestamp: "2026-04-24T21:30:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:30:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:36Z" name: message-dumper-raw-66937-predictor-86db465969-zxjgs.18a9685392bab088 namespace: kserve-ci-e2e-test resourceVersion: "16720" uid: 21d38b39-b2a9-42bb-b7fc-b177987791dc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-66937-predictor-86db465969 namespace: kserve-ci-e2e-test resourceVersion: "15723" uid: 073f461d-d26d-48fa-ac6c-918b715af204 kind: Event lastTimestamp: "2026-04-24T21:28:50Z" message: 'Created pod: message-dumper-raw-66937-predictor-86db465969-zxjgs' metadata: creationTimestamp: "2026-04-24T21:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:28:50Z" name: message-dumper-raw-66937-predictor-86db465969.18a9683b0aea9828 namespace: kserve-ci-e2e-test resourceVersion: "15747" uid: 95e2d0a2-7d47-4ecc-9bcc-f4963464350d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-66937-predictor namespace: kserve-ci-e2e-test resourceVersion: "15720" uid: 44393140-4ab5-43a6-8785-150b05cfb6e7 kind: Event lastTimestamp: "2026-04-24T21:28:50Z" message: Scaled up replica set message-dumper-raw-66937-predictor-86db465969 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:28:50Z" name: message-dumper-raw-66937-predictor.18a9683b077e1102 namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: 2261391f-0cae-4c8c-b874-eeae119e052e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:29:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-66937-predictor namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: f1674b2e-dee0-48d0-9177-53236b7651c5 kind: Event lastTimestamp: "2026-04-24T21:29:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:29:20Z" name: message-dumper-raw-66937-predictor.18a9683e8683f716 namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: af16c1c1-122f-4b7b-803a-ff3a6bf364b4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:29:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-66937-predictor namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: f1674b2e-dee0-48d0-9177-53236b7651c5 kind: Event lastTimestamp: "2026-04-24T21:29:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:29:20Z" name: message-dumper-raw-66937-predictor.18a9683e86e6af0e namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: b0a04379-b8d5-44ea-baf9-67a9f0cb39a0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-66937-predictor namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: f1674b2e-dee0-48d0-9177-53236b7651c5 kind: Event lastTimestamp: "2026-04-24T21:29:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:29:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:29:35Z" name: message-dumper-raw-66937-predictor.18a9684584d5909a namespace: kserve-ci-e2e-test resourceVersion: "16206" uid: 751e6e4c-72cc-4e0c-a9c7-f11d2d2b2e99 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-66937-predictor namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: f1674b2e-dee0-48d0-9177-53236b7651c5 kind: Event lastTimestamp: "2026-04-24T21:29:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:29:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:29:35Z" name: message-dumper-raw-66937-predictor.18a96845857023b6 namespace: kserve-ci-e2e-test resourceVersion: "16208" uid: 6edd083e-cb42-4a40-928a-526c0b40c3e6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:28:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-66937 namespace: kserve-ci-e2e-test resourceVersion: "15708" uid: c2829ab5-4fbe-4465-a579-478b71095b0d kind: Event lastTimestamp: "2026-04-24T21:29:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:29:01Z" name: message-dumper-raw-66937.18a9683b07badd87 namespace: kserve-ci-e2e-test resourceVersion: "15885" uid: a1d730f6-7f22-4b5f-84b5-83acb931a86e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-66937 namespace: kserve-ci-e2e-test resourceVersion: "15708" uid: c2829ab5-4fbe-4465-a579-478b71095b0d kind: Event lastTimestamp: "2026-04-24T21:28:50Z" message: 'Failed to update status for InferenceService "message-dumper-raw-66937": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-66937": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:28:50Z" name: message-dumper-raw-66937.18a9683b083a9109 namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 9ec61d14-b830-48cf-9357-534d5ff995dc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-66937 namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: c2829ab5-4fbe-4465-a579-478b71095b0d kind: Event lastTimestamp: "2026-04-24T21:29:01Z" message: InferenceService [message-dumper-raw-66937] is Ready metadata: creationTimestamp: "2026-04-24T21:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:29:01Z" name: message-dumper-raw-66937.18a9683da0410e48 namespace: kserve-ci-e2e-test resourceVersion: "15883" uid: bb751cd4-6e7e-4304-9167-179738abca3e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:27:00.246965Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-da412-6cd7bcff7-rvqs2 namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: 0b7862b8-8f4d-4383-a836-02b97d0c81a1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-da412-6cd7bcff7-rvqs2 to ip-10-0-141-46.ec2.internal metadata: creationTimestamp: "2026-04-24T21:27:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:27:00Z" name: model-chainer-raw-da412-6cd7bcff7-rvqs2.18a968214f3ce98e namespace: kserve-ci-e2e-test resourceVersion: "14603" uid: 89e8caf0-eaac-42f5-bdb7-9c7b5215aa73 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6b5fbb9956-pzrgr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:00Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-da412-6cd7bcff7-rvqs2 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 0b7862b8-8f4d-4383-a836-02b97d0c81a1 kind: Event lastTimestamp: "2026-04-24T21:27:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-da412-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:00Z" name: model-chainer-raw-da412-6cd7bcff7-rvqs2.18a968215901993b namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: a6608f75-8c52-495f-b0bd-9daf3c4a808a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-46.ec2.internal source: component: kubelet host: ip-10-0-141-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:01Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-da412-6cd7bcff7-rvqs2 namespace: kserve-ci-e2e-test resourceVersion: "14606" uid: 0b7862b8-8f4d-4383-a836-02b97d0c81a1 kind: Event lastTimestamp: "2026-04-24T21:27:01Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:27:01Z" name: model-chainer-raw-da412-6cd7bcff7-rvqs2.18a968218c1d99f2 namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: ae4ea065-cd58-427d-ae5f-b0d7dcc339d6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-da412} kind: Pod name: model-chainer-raw-da412-6cd7bcff7-rvqs2 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 0b7862b8-8f4d-4383-a836-02b97d0c81a1 kind: Event lastTimestamp: "2026-04-24T21:27:01Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:f2021b9f79dbd151723ae4c386dd28bedd0fff23f6daa0825bdc58598fc71887" metadata: creationTimestamp: "2026-04-24T21:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:01Z" name: model-chainer-raw-da412-6cd7bcff7-rvqs2.18a968218daf51c1 namespace: kserve-ci-e2e-test resourceVersion: "14622" uid: a0347832-2a91-4803-a323-7ce3556424ce reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-46.ec2.internal source: component: kubelet host: ip-10-0-141-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-da412} kind: Pod name: model-chainer-raw-da412-6cd7bcff7-rvqs2 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 0b7862b8-8f4d-4383-a836-02b97d0c81a1 kind: Event lastTimestamp: "2026-04-24T21:27:03Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:f2021b9f79dbd151723ae4c386dd28bedd0fff23f6daa0825bdc58598fc71887" in 2.139s (2.139s including waiting). Image size: 216340060 bytes.' metadata: creationTimestamp: "2026-04-24T21:27:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:03Z" name: model-chainer-raw-da412-6cd7bcff7-rvqs2.18a968220d3a31aa namespace: kserve-ci-e2e-test resourceVersion: "14640" uid: 31c98c73-d75a-4aae-a54e-f4b2bc8b2825 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-46.ec2.internal source: component: kubelet host: ip-10-0-141-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-da412} kind: Pod name: model-chainer-raw-da412-6cd7bcff7-rvqs2 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 0b7862b8-8f4d-4383-a836-02b97d0c81a1 kind: Event lastTimestamp: "2026-04-24T21:27:03Z" message: 'Created container: model-chainer-raw-da412' metadata: creationTimestamp: "2026-04-24T21:27:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:03Z" name: model-chainer-raw-da412-6cd7bcff7-rvqs2.18a96822112c3166 namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: 9fa1be57-caeb-49dd-a970-861dd950f185 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-46.ec2.internal source: component: kubelet host: ip-10-0-141-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-da412} kind: Pod name: model-chainer-raw-da412-6cd7bcff7-rvqs2 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 0b7862b8-8f4d-4383-a836-02b97d0c81a1 kind: Event lastTimestamp: "2026-04-24T21:27:03Z" message: Started container model-chainer-raw-da412 metadata: creationTimestamp: "2026-04-24T21:27:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:03Z" name: model-chainer-raw-da412-6cd7bcff7-rvqs2.18a96822119fa5f9 namespace: kserve-ci-e2e-test resourceVersion: "14643" uid: 4ec74855-bc14-4c92-bddd-9f8e0d7e1edb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-46.ec2.internal source: component: kubelet host: ip-10-0-141-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-da412} kind: Pod name: model-chainer-raw-da412-6cd7bcff7-rvqs2 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 0b7862b8-8f4d-4383-a836-02b97d0c81a1 kind: Event lastTimestamp: "2026-04-24T21:27:10Z" message: Stopping container model-chainer-raw-da412 metadata: creationTimestamp: "2026-04-24T21:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:10Z" name: model-chainer-raw-da412-6cd7bcff7-rvqs2.18a96823a60e6ff6 namespace: kserve-ci-e2e-test resourceVersion: "14722" uid: 0fe3fa4a-5e8a-4c45-98f7-bbbe99b0b55d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-46.ec2.internal source: component: kubelet host: ip-10-0-141-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-da412} kind: Pod name: model-chainer-raw-da412-6cd7bcff7-rvqs2 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 0b7862b8-8f4d-4383-a836-02b97d0c81a1 kind: Event lastTimestamp: "2026-04-24T21:27:40Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:27:40Z" name: model-chainer-raw-da412-6cd7bcff7-rvqs2.18a96824c1e76ec9 namespace: kserve-ci-e2e-test resourceVersion: "15116" uid: eb73f131-f96c-4170-93b1-41317bdbdb41 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-46.ec2.internal source: component: kubelet host: ip-10-0-141-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-da412-6cd7bcff7 namespace: kserve-ci-e2e-test resourceVersion: "14594" uid: dae3e846-b89a-44c0-9766-2235d17c4148 kind: Event lastTimestamp: "2026-04-24T21:27:00Z" message: 'Created pod: model-chainer-raw-da412-6cd7bcff7-rvqs2' metadata: creationTimestamp: "2026-04-24T21:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:27:00Z" name: model-chainer-raw-da412-6cd7bcff7.18a968214ec458c3 namespace: kserve-ci-e2e-test resourceVersion: "14599" uid: 01721b4d-e5d8-48c7-8148-6fb84f73cd3f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-da412 namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: f9081a11-bc82-44c3-9359-886311e79160 kind: Event lastTimestamp: "2026-04-24T21:27:00Z" message: Scaled up replica set model-chainer-raw-da412-6cd7bcff7 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:27:00Z" name: model-chainer-raw-da412.18a968214de90b1d namespace: kserve-ci-e2e-test resourceVersion: "14595" uid: 310dacf6-5abf-48cb-b9f6-f72c5b8c6715 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:27:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-da412 namespace: kserve-ci-e2e-test resourceVersion: "14608" uid: a92f71a7-c7e9-4a88-bff8-309df5417df0 kind: Event lastTimestamp: "2026-04-24T21:27:00Z" message: InferenceGraph [model-chainer-raw-da412] is Ready metadata: creationTimestamp: "2026-04-24T21:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:27:00Z" name: model-chainer-raw-da412.18a9682151d845c0 namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: c9c2f152-cbe3-40e2-b149-f473c368ff96 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:28:40.536061Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-07ea4-759d56d8b5-jpmwh namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: 5b958c6d-4937-418e-80f3-92a4d099c830 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-07ea4-759d56d8b5-jpmwh to ip-10-0-141-46.ec2.internal metadata: creationTimestamp: "2026-04-24T21:28:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:28:40Z" name: model-chainer-raw-hpa-07ea4-759d56d8b5-jpmwh.18a96838a8ef1873 namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: 63aafe1a-18cb-4d0c-bfcf-f6bd408042b7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6b5fbb9956-pzrgr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:40Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-07ea4-759d56d8b5-jpmwh namespace: kserve-ci-e2e-test resourceVersion: "15586" uid: 5b958c6d-4937-418e-80f3-92a4d099c830 kind: Event lastTimestamp: "2026-04-24T21:28:40Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-07ea4-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:40Z" name: model-chainer-raw-hpa-07ea4-759d56d8b5-jpmwh.18a96838b2518cd4 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: c90f9f00-f616-4c9f-abb4-c3c7bb0003cc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-46.ec2.internal source: component: kubelet host: ip-10-0-141-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:41Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-07ea4-759d56d8b5-jpmwh namespace: kserve-ci-e2e-test resourceVersion: "15593" uid: 5b958c6d-4937-418e-80f3-92a4d099c830 kind: Event lastTimestamp: "2026-04-24T21:28:41Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:28:41Z" name: model-chainer-raw-hpa-07ea4-759d56d8b5-jpmwh.18a96838e5f0224a namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: 44526e57-cc50-43d6-a4d5-11a1adf72d09 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-07ea4} kind: Pod name: model-chainer-raw-hpa-07ea4-759d56d8b5-jpmwh namespace: kserve-ci-e2e-test resourceVersion: "15586" uid: 5b958c6d-4937-418e-80f3-92a4d099c830 kind: Event lastTimestamp: "2026-04-24T21:28:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f2021b9f79dbd151723ae4c386dd28bedd0fff23f6daa0825bdc58598fc71887" already present on machine metadata: creationTimestamp: "2026-04-24T21:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:41Z" name: model-chainer-raw-hpa-07ea4-759d56d8b5-jpmwh.18a96838e71f94de namespace: kserve-ci-e2e-test resourceVersion: "15613" uid: b6788862-45c0-4588-90d9-d4b88ede85f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-46.ec2.internal source: component: kubelet host: ip-10-0-141-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-07ea4} kind: Pod name: model-chainer-raw-hpa-07ea4-759d56d8b5-jpmwh namespace: kserve-ci-e2e-test resourceVersion: "15586" uid: 5b958c6d-4937-418e-80f3-92a4d099c830 kind: Event lastTimestamp: "2026-04-24T21:28:41Z" message: 'Created container: model-chainer-raw-hpa-07ea4' metadata: creationTimestamp: "2026-04-24T21:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:41Z" name: model-chainer-raw-hpa-07ea4-759d56d8b5-jpmwh.18a96838e9ded2e9 namespace: kserve-ci-e2e-test resourceVersion: "15614" uid: 21a59ad0-56af-46c7-a3e3-57f1d90e670c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-46.ec2.internal source: component: kubelet host: ip-10-0-141-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-07ea4} kind: Pod name: model-chainer-raw-hpa-07ea4-759d56d8b5-jpmwh namespace: kserve-ci-e2e-test resourceVersion: "15586" uid: 5b958c6d-4937-418e-80f3-92a4d099c830 kind: Event lastTimestamp: "2026-04-24T21:28:41Z" message: Started container model-chainer-raw-hpa-07ea4 metadata: creationTimestamp: "2026-04-24T21:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:41Z" name: model-chainer-raw-hpa-07ea4-759d56d8b5-jpmwh.18a96838ea507067 namespace: kserve-ci-e2e-test resourceVersion: "15615" uid: 9bd26d55-3531-49ea-a94b-5102cde2efaa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-46.ec2.internal source: component: kubelet host: ip-10-0-141-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-07ea4} kind: Pod name: model-chainer-raw-hpa-07ea4-759d56d8b5-jpmwh namespace: kserve-ci-e2e-test resourceVersion: "15586" uid: 5b958c6d-4937-418e-80f3-92a4d099c830 kind: Event lastTimestamp: "2026-04-24T21:28:50Z" message: Stopping container model-chainer-raw-hpa-07ea4 metadata: creationTimestamp: "2026-04-24T21:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:50Z" name: model-chainer-raw-hpa-07ea4-759d56d8b5-jpmwh.18a9683b001ea6a6 namespace: kserve-ci-e2e-test resourceVersion: "15704" uid: 8b303321-cd94-4035-a548-d96d23de4372 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-46.ec2.internal source: component: kubelet host: ip-10-0-141-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:28:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-07ea4} kind: Pod name: model-chainer-raw-hpa-07ea4-759d56d8b5-jpmwh namespace: kserve-ci-e2e-test resourceVersion: "15586" uid: 5b958c6d-4937-418e-80f3-92a4d099c830 kind: Event lastTimestamp: "2026-04-24T21:29:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:18Z" name: model-chainer-raw-hpa-07ea4-759d56d8b5-jpmwh.18a9683ba5aad02e namespace: kserve-ci-e2e-test resourceVersion: "16081" uid: 5953a073-375d-4869-acb9-f8a7c180eb26 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-46.ec2.internal source: component: kubelet host: ip-10-0-141-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-07ea4-759d56d8b5 namespace: kserve-ci-e2e-test resourceVersion: "15581" uid: aa224610-0f3a-4f74-a5f8-d3e60dad5f6b kind: Event lastTimestamp: "2026-04-24T21:28:40Z" message: 'Created pod: model-chainer-raw-hpa-07ea4-759d56d8b5-jpmwh' metadata: creationTimestamp: "2026-04-24T21:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:28:40Z" name: model-chainer-raw-hpa-07ea4-759d56d8b5.18a96838a8872183 namespace: kserve-ci-e2e-test resourceVersion: "15585" uid: 51b07fce-9c13-4160-a493-6b33271f75e4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-07ea4 namespace: kserve-ci-e2e-test resourceVersion: "15579" uid: 88fbec92-78f9-45ba-b3b3-074dcb7aab34 kind: Event lastTimestamp: "2026-04-24T21:28:40Z" message: Scaled up replica set model-chainer-raw-hpa-07ea4-759d56d8b5 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:28:40Z" name: model-chainer-raw-hpa-07ea4.18a96838a7c6dab4 namespace: kserve-ci-e2e-test resourceVersion: "15583" uid: 307c2850-9d8f-4e74-8e90-d0445ca4ed96 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-07ea4 namespace: kserve-ci-e2e-test resourceVersion: "15596" uid: 894562d9-00eb-4f81-9235-170e1b6c4b3b kind: Event lastTimestamp: "2026-04-24T21:28:40Z" message: InferenceGraph [model-chainer-raw-hpa-07ea4] is Ready metadata: creationTimestamp: "2026-04-24T21:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:28:40Z" name: model-chainer-raw-hpa-07ea4.18a96838aba98fc2 namespace: kserve-ci-e2e-test resourceVersion: "15598" uid: 318571ae-8c6e-4941-9ae5-441855aad814 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:34:54.802061Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4 namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: a8e70f70-f47d-48b8-88d6-5d064bbde0d1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-d6223-predictor-7ccc8855db-kvsx4 to ip-10-0-132-118.ec2.internal metadata: creationTimestamp: "2026-04-24T21:34:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:34:54Z" name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4.18a9688fcced1711 namespace: kserve-ci-e2e-test resourceVersion: "18907" uid: 8d55fce8-fa76-418f-8067-ccda69f96b78 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6b5fbb9956-pzrgr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:55Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4 namespace: kserve-ci-e2e-test resourceVersion: "18914" uid: a8e70f70-f47d-48b8-88d6-5d064bbde0d1 kind: Event lastTimestamp: "2026-04-24T21:34:55Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:34:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:34:55Z" name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4.18a9688fe659217f namespace: kserve-ci-e2e-test resourceVersion: "18924" uid: cf11a902-ca32-4644-89ed-526f370aea78 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4 namespace: kserve-ci-e2e-test resourceVersion: "18905" uid: a8e70f70-f47d-48b8-88d6-5d064bbde0d1 kind: Event lastTimestamp: "2026-04-24T21:34:55Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:85f85c16414082de64de782f602228652c10c5bace08b518e803a7e911c62e5e" already present on machine metadata: creationTimestamp: "2026-04-24T21:34:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:55Z" name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4.18a9688fe7d8a25e namespace: kserve-ci-e2e-test resourceVersion: "18927" uid: fbe5a234-9638-4b54-a4ce-4471783b8e20 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4 namespace: kserve-ci-e2e-test resourceVersion: "18905" uid: a8e70f70-f47d-48b8-88d6-5d064bbde0d1 kind: Event lastTimestamp: "2026-04-24T21:34:55Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:34:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:55Z" name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4.18a9688febdc39d0 namespace: kserve-ci-e2e-test resourceVersion: "18930" uid: e8c0fa7f-86fc-496f-adcd-c8e385bca70b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4 namespace: kserve-ci-e2e-test resourceVersion: "18905" uid: a8e70f70-f47d-48b8-88d6-5d064bbde0d1 kind: Event lastTimestamp: "2026-04-24T21:34:55Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:34:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:55Z" name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4.18a9688fec4f93a1 namespace: kserve-ci-e2e-test resourceVersion: "18931" uid: 0e6c4b94-e28d-43d0-9905-d8f1e2704ce1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4 namespace: kserve-ci-e2e-test resourceVersion: "18905" uid: a8e70f70-f47d-48b8-88d6-5d064bbde0d1 kind: Event lastTimestamp: "2026-04-24T21:34:59Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:34:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:59Z" name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4.18a96890fe2882ac namespace: kserve-ci-e2e-test resourceVersion: "18971" uid: d83c819e-9e34-4f4f-94b9-5f3bea3f34f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4 namespace: kserve-ci-e2e-test resourceVersion: "18905" uid: a8e70f70-f47d-48b8-88d6-5d064bbde0d1 kind: Event lastTimestamp: "2026-04-24T21:34:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:34:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:59Z" name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4.18a9689100d73d19 namespace: kserve-ci-e2e-test resourceVersion: "18974" uid: 5390d96b-e4c0-4688-b293-1c06056a6771 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4 namespace: kserve-ci-e2e-test resourceVersion: "18905" uid: a8e70f70-f47d-48b8-88d6-5d064bbde0d1 kind: Event lastTimestamp: "2026-04-24T21:34:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:34:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:59Z" name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4.18a9689101488ad9 namespace: kserve-ci-e2e-test resourceVersion: "18976" uid: 31db89dd-272d-4094-98d9-2431241c817a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4 namespace: kserve-ci-e2e-test resourceVersion: "18905" uid: a8e70f70-f47d-48b8-88d6-5d064bbde0d1 kind: Event lastTimestamp: "2026-04-24T21:34:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:34:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:59Z" name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4.18a968910154b997 namespace: kserve-ci-e2e-test resourceVersion: "18978" uid: fdf6fe50-bb4c-4972-a0b3-cf9d811a2008 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4 namespace: kserve-ci-e2e-test resourceVersion: "18905" uid: a8e70f70-f47d-48b8-88d6-5d064bbde0d1 kind: Event lastTimestamp: "2026-04-24T21:35:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:35:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:00Z" name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4.18a96891044a146f namespace: kserve-ci-e2e-test resourceVersion: "18980" uid: 87c25e03-5c25-4735-8cb5-66ad566cd8fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4 namespace: kserve-ci-e2e-test resourceVersion: "18905" uid: a8e70f70-f47d-48b8-88d6-5d064bbde0d1 kind: Event lastTimestamp: "2026-04-24T21:35:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:35:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:00Z" name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4.18a9689104b9e8ca namespace: kserve-ci-e2e-test resourceVersion: "18981" uid: e301a96c-a9d5-453b-be4f-7d120327f6f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:35:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4 namespace: kserve-ci-e2e-test resourceVersion: "18905" uid: a8e70f70-f47d-48b8-88d6-5d064bbde0d1 kind: Event lastTimestamp: "2026-04-24T21:36:16Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:35:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:16Z" name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4.18a968913a0cc2b3 namespace: kserve-ci-e2e-test resourceVersion: "19606" uid: beeaeba4-deba-435d-ad0d-bf9a4922b366 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4 namespace: kserve-ci-e2e-test resourceVersion: "18905" uid: a8e70f70-f47d-48b8-88d6-5d064bbde0d1 kind: Event lastTimestamp: "2026-04-24T21:36:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:14Z" name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4.18a968a27380c047 namespace: kserve-ci-e2e-test resourceVersion: "19545" uid: 25e2d8c3-e6f3-4f29-a032-471566d57009 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4 namespace: kserve-ci-e2e-test resourceVersion: "18905" uid: a8e70f70-f47d-48b8-88d6-5d064bbde0d1 kind: Event lastTimestamp: "2026-04-24T21:36:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:14Z" name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4.18a968a2738144d1 namespace: kserve-ci-e2e-test resourceVersion: "19548" uid: 1eabe1c6-6e18-4f74-8927-64cfe9480875 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4 namespace: kserve-ci-e2e-test resourceVersion: "18905" uid: a8e70f70-f47d-48b8-88d6-5d064bbde0d1 kind: Event lastTimestamp: "2026-04-24T21:36:16Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:36:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:16Z" name: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4.18a968a2ec251b21 namespace: kserve-ci-e2e-test resourceVersion: "19605" uid: 0d0045b2-262a-4a43-aa20-ea050dc3136b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-d6223-predictor-7ccc8855db namespace: kserve-ci-e2e-test resourceVersion: "18890" uid: 773a79e6-5d2c-48e2-8f44-504aa754d66b kind: Event lastTimestamp: "2026-04-24T21:34:54Z" message: 'Created pod: raw-sklearn-d6223-predictor-7ccc8855db-kvsx4' metadata: creationTimestamp: "2026-04-24T21:34:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:54Z" name: raw-sklearn-d6223-predictor-7ccc8855db.18a9688fcc7b4bec namespace: kserve-ci-e2e-test resourceVersion: "18904" uid: 2db6490e-995b-494c-bf84-ebc9bd646ae8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-d6223-predictor namespace: kserve-ci-e2e-test resourceVersion: "18888" uid: 0ce9fead-7142-4ab4-9eac-6144e1567b93 kind: Event lastTimestamp: "2026-04-24T21:34:54Z" message: Scaled up replica set raw-sklearn-d6223-predictor-7ccc8855db from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:34:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:54Z" name: raw-sklearn-d6223-predictor.18a9688fc956c328 namespace: kserve-ci-e2e-test resourceVersion: "18892" uid: d4d7d93f-bb0f-42b1-920c-46c92a73a05a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:35:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-d6223-predictor namespace: kserve-ci-e2e-test resourceVersion: "18889" uid: 45a0507f-b443-4c4f-a954-47406bf2a660 kind: Event lastTimestamp: "2026-04-24T21:35:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:35:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:39Z" name: raw-sklearn-d6223-predictor.18a968934837cc62 namespace: kserve-ci-e2e-test resourceVersion: "19260" uid: 618fefbc-3b95-442b-9b7b-aa662d99acb3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:35:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-d6223-predictor namespace: kserve-ci-e2e-test resourceVersion: "18889" uid: 45a0507f-b443-4c4f-a954-47406bf2a660 kind: Event lastTimestamp: "2026-04-24T21:35:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:35:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:39Z" name: raw-sklearn-d6223-predictor.18a96893489a7567 namespace: kserve-ci-e2e-test resourceVersion: "19261" uid: fa406c09-5de0-44a2-bcb6-420129653092 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:35:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-d6223-predictor namespace: kserve-ci-e2e-test resourceVersion: "19052" uid: 45a0507f-b443-4c4f-a954-47406bf2a660 kind: Event lastTimestamp: "2026-04-24T21:36:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:35:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:09Z" name: raw-sklearn-d6223-predictor.18a9689dc53055e7 namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: d0c33424-3152-42a5-9074-0e7040363aa2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:35:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-d6223-predictor namespace: kserve-ci-e2e-test resourceVersion: "19052" uid: 45a0507f-b443-4c4f-a954-47406bf2a660 kind: Event lastTimestamp: "2026-04-24T21:36:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:35:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:09Z" name: raw-sklearn-d6223-predictor.18a9689dc579d277 namespace: kserve-ci-e2e-test resourceVersion: "19482" uid: 3d9d0b46-fda9-4c89-9119-8f505299fa89 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T21:34:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-d6223 namespace: kserve-ci-e2e-test resourceVersion: "18878" uid: 46153ed8-8ec0-4649-bc91-4fdc4ad5c788 kind: Event lastTimestamp: "2026-04-24T21:36:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:34:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:36:07Z" name: raw-sklearn-d6223.18a9688fc9311e40 namespace: kserve-ci-e2e-test resourceVersion: "19460" uid: 5a3954a3-0495-4701-9b24-3306cf71d9cb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-d6223 namespace: kserve-ci-e2e-test resourceVersion: "18878" uid: 46153ed8-8ec0-4649-bc91-4fdc4ad5c788 kind: Event lastTimestamp: "2026-04-24T21:34:54Z" message: 'Failed to update status for InferenceService "raw-sklearn-d6223": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-d6223": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:34:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:34:54Z" name: raw-sklearn-d6223.18a9688fc98e03b0 namespace: kserve-ci-e2e-test resourceVersion: "18894" uid: a721cc0d-c854-450f-a433-82e73218ade2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-d6223 namespace: kserve-ci-e2e-test resourceVersion: "19458" uid: 46153ed8-8ec0-4649-bc91-4fdc4ad5c788 kind: Event lastTimestamp: "2026-04-24T21:36:07Z" message: InferenceService [raw-sklearn-d6223] is Ready metadata: creationTimestamp: "2026-04-24T21:36:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:36:07Z" name: raw-sklearn-d6223.18a968a09d071153 namespace: kserve-ci-e2e-test resourceVersion: "19459" uid: 569b6876-3fb2-4208-bfaf-4407a09182bf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:36:14.990853Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff namespace: kserve-ci-e2e-test resourceVersion: "19561" uid: 8547082c-6adf-40d2-8dce-e15eb63349b2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff to ip-10-0-132-118.ec2.internal metadata: creationTimestamp: "2026-04-24T21:36:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:36:14Z" name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff.18a968a2788cea37 namespace: kserve-ci-e2e-test resourceVersion: "19567" uid: 9f3530f8-5fa6-4913-a48f-e204ce8c8ce3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6b5fbb9956-pzrgr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:15Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 8547082c-6adf-40d2-8dce-e15eb63349b2 kind: Event lastTimestamp: "2026-04-24T21:36:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-runtime-8f851-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:36:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:15Z" name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff.18a968a28369faac namespace: kserve-ci-e2e-test resourceVersion: "19576" uid: 4f94095e-6061-4e83-9ff0-1fb37a935507 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:16Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff namespace: kserve-ci-e2e-test resourceVersion: "19571" uid: 8547082c-6adf-40d2-8dce-e15eb63349b2 kind: Event lastTimestamp: "2026-04-24T21:36:16Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:36:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:36:16Z" name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff.18a968a2b51ed9c7 namespace: kserve-ci-e2e-test resourceVersion: "19586" uid: 89a3d8ab-061f-46ce-a63e-ce7ac49009a4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 8547082c-6adf-40d2-8dce-e15eb63349b2 kind: Event lastTimestamp: "2026-04-24T21:36:16Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:85f85c16414082de64de782f602228652c10c5bace08b518e803a7e911c62e5e" already present on machine metadata: creationTimestamp: "2026-04-24T21:36:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:16Z" name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff.18a968a2b665cd52 namespace: kserve-ci-e2e-test resourceVersion: "19588" uid: 1f8d67d8-66f3-4507-809e-dec728087ef4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 8547082c-6adf-40d2-8dce-e15eb63349b2 kind: Event lastTimestamp: "2026-04-24T21:36:16Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:36:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:16Z" name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff.18a968a2b9a32732 namespace: kserve-ci-e2e-test resourceVersion: "19590" uid: b5dab10e-e995-4f82-9096-210b9c45dd1b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 8547082c-6adf-40d2-8dce-e15eb63349b2 kind: Event lastTimestamp: "2026-04-24T21:36:16Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:36:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:16Z" name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff.18a968a2ba0f49bc namespace: kserve-ci-e2e-test resourceVersion: "19591" uid: 774fcac2-fd17-4d29-a295-24e556f83666 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 8547082c-6adf-40d2-8dce-e15eb63349b2 kind: Event lastTimestamp: "2026-04-24T21:36:20Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:20Z" name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff.18a968a3ad0d23df namespace: kserve-ci-e2e-test resourceVersion: "19631" uid: 53704095-a77c-4985-9f19-4660e81d869b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 8547082c-6adf-40d2-8dce-e15eb63349b2 kind: Event lastTimestamp: "2026-04-24T21:36:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:20Z" name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff.18a968a3b006acef namespace: kserve-ci-e2e-test resourceVersion: "19633" uid: 6b1c9af1-37df-4e78-a86f-eb6be1413e17 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 8547082c-6adf-40d2-8dce-e15eb63349b2 kind: Event lastTimestamp: "2026-04-24T21:36:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:20Z" name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff.18a968a3b077113c namespace: kserve-ci-e2e-test resourceVersion: "19636" uid: fd426d6d-fda7-41c8-b894-3a7f9a04a26f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 8547082c-6adf-40d2-8dce-e15eb63349b2 kind: Event lastTimestamp: "2026-04-24T21:36:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:20Z" name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff.18a968a3b0819e1f namespace: kserve-ci-e2e-test resourceVersion: "19637" uid: ecbf2875-e7b9-43ee-a34d-2ec49476366e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 8547082c-6adf-40d2-8dce-e15eb63349b2 kind: Event lastTimestamp: "2026-04-24T21:36:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:20Z" name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff.18a968a3b37d9122 namespace: kserve-ci-e2e-test resourceVersion: "19639" uid: 074b29ee-7f66-4766-8a9e-49a76f7350b9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 8547082c-6adf-40d2-8dce-e15eb63349b2 kind: Event lastTimestamp: "2026-04-24T21:36:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:20Z" name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff.18a968a3b3ee0d6c namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: eda369c4-80d5-47de-85e7-7d9de9b1f125 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:36:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 8547082c-6adf-40d2-8dce-e15eb63349b2 kind: Event lastTimestamp: "2026-04-24T21:37:37Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:36:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:37Z" name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff.18a968a3e8f42445 namespace: kserve-ci-e2e-test resourceVersion: "20245" uid: ef042645-aa8e-435a-b9e1-60424520d205 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 8547082c-6adf-40d2-8dce-e15eb63349b2 kind: Event lastTimestamp: "2026-04-24T21:37:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:35Z" name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff.18a968b520b034ed namespace: kserve-ci-e2e-test resourceVersion: "20195" uid: 81987190-7d5c-4f8e-bf8c-e5c5daae6088 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 8547082c-6adf-40d2-8dce-e15eb63349b2 kind: Event lastTimestamp: "2026-04-24T21:37:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:35Z" name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff.18a968b520b0cb34 namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: 578a4394-544c-416b-820a-ebee89797d14 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 8547082c-6adf-40d2-8dce-e15eb63349b2 kind: Event lastTimestamp: "2026-04-24T21:37:37Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:37Z" name: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff.18a968b59b103ab8 namespace: kserve-ci-e2e-test resourceVersion: "20244" uid: 2c6cab0c-2384-44d3-9c2f-64a9069b7eac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-118.ec2.internal source: component: kubelet host: ip-10-0-132-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-8f851-predictor-6ccb889f98 namespace: kserve-ci-e2e-test resourceVersion: "19551" uid: ff38e393-c9f7-42cf-b2ec-c8fed4df8490 kind: Event lastTimestamp: "2026-04-24T21:36:14Z" message: 'Created pod: raw-sklearn-runtime-8f851-predictor-6ccb889f98-5bcff' metadata: creationTimestamp: "2026-04-24T21:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:14Z" name: raw-sklearn-runtime-8f851-predictor-6ccb889f98.18a968a2782878e4 namespace: kserve-ci-e2e-test resourceVersion: "19564" uid: 76d91ac1-4dd5-41ea-b531-9b7936ddb6f5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-8f851-predictor namespace: kserve-ci-e2e-test resourceVersion: "19549" uid: c0f11d8e-b6f5-4492-9510-d31f6af34a61 kind: Event lastTimestamp: "2026-04-24T21:36:14Z" message: Scaled up replica set raw-sklearn-runtime-8f851-predictor-6ccb889f98 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:14Z" name: raw-sklearn-runtime-8f851-predictor.18a968a275069615 namespace: kserve-ci-e2e-test resourceVersion: "19554" uid: 4e9f9a65-bbf2-4263-8950-57f58bf74c12 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:36:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-8f851-predictor namespace: kserve-ci-e2e-test resourceVersion: "19550" uid: 343ca447-7980-482f-b535-ec569862642e kind: Event lastTimestamp: "2026-04-24T21:36:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:36:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:59Z" name: raw-sklearn-runtime-8f851-predictor.18a968a5f397d7d0 namespace: kserve-ci-e2e-test resourceVersion: "19919" uid: 55e3786e-942e-4af4-adca-c635cfc0c210 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:36:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-8f851-predictor namespace: kserve-ci-e2e-test resourceVersion: "19550" uid: 343ca447-7980-482f-b535-ec569862642e kind: Event lastTimestamp: "2026-04-24T21:36:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:36:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:59Z" name: raw-sklearn-runtime-8f851-predictor.18a968a5f3f897b7 namespace: kserve-ci-e2e-test resourceVersion: "19920" uid: 7a984b8d-88ab-4f8b-8fdf-840f56bb5c36 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:37:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-8f851-predictor namespace: kserve-ci-e2e-test resourceVersion: "19710" uid: 343ca447-7980-482f-b535-ec569862642e kind: Event lastTimestamp: "2026-04-24T21:37:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:30Z" name: raw-sklearn-runtime-8f851-predictor.18a968b07102eb60 namespace: kserve-ci-e2e-test resourceVersion: "20141" uid: a926eae4-bbe9-4d79-9511-0c4e0bdda37d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:37:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-8f851-predictor namespace: kserve-ci-e2e-test resourceVersion: "19710" uid: 343ca447-7980-482f-b535-ec569862642e kind: Event lastTimestamp: "2026-04-24T21:37:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:30Z" name: raw-sklearn-runtime-8f851-predictor.18a968b0716dc28d namespace: kserve-ci-e2e-test resourceVersion: "20142" uid: 22aa6f35-1733-47cf-8b1d-021a12fbee69 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T21:36:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-8f851 namespace: kserve-ci-e2e-test resourceVersion: "19521" uid: 1bf9be54-0225-4276-9531-48126b427eba kind: Event lastTimestamp: "2026-04-24T21:37:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:37:27Z" name: raw-sklearn-runtime-8f851.18a968a274e09350 namespace: kserve-ci-e2e-test resourceVersion: "20119" uid: 1ddb2453-4c09-4336-b002-28578561443a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-8f851 namespace: kserve-ci-e2e-test resourceVersion: "19521" uid: 1bf9be54-0225-4276-9531-48126b427eba kind: Event lastTimestamp: "2026-04-24T21:36:14Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-8f851": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-8f851": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:36:14Z" name: raw-sklearn-runtime-8f851.18a968a2754fcbcb namespace: kserve-ci-e2e-test resourceVersion: "19556" uid: 4ea1997d-19a7-41ac-903e-b98abfadf4a9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-8f851 namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: 1bf9be54-0225-4276-9531-48126b427eba kind: Event lastTimestamp: "2026-04-24T21:37:27Z" message: InferenceService [raw-sklearn-runtime-8f851] is Ready metadata: creationTimestamp: "2026-04-24T21:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:37:27Z" name: raw-sklearn-runtime-8f851.18a968b34d074289 namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: f4add941-81cf-4f85-83ac-190206aaa049 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "20384"