--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:45:21.887782Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-e9425b-predictor-75f7ff4474-bpdjx namespace: kserve-ci-e2e-test resourceVersion: "18669" uid: 4a20173a-6851-4bc3-b559-cfa77ead9848 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-e9425b-predictor-75f7ff4474-bpdjx to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-24T21:45:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:45:21Z" name: isvc-init-fail-e9425b-predictor-75f7ff4474-bpdjx.18a96921ce25bf49 namespace: kserve-ci-e2e-test resourceVersion: "18673" uid: 989e4fe6-567d-49c3-affe-36ab91a4442c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6f5f9cfdf5-qjgtv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:22Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-e9425b-predictor-75f7ff4474-bpdjx namespace: kserve-ci-e2e-test resourceVersion: "18671" uid: 4a20173a-6851-4bc3-b559-cfa77ead9848 kind: Event lastTimestamp: "2026-04-24T21:45:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-init-fail-e9425b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:22Z" name: isvc-init-fail-e9425b-predictor-75f7ff4474-bpdjx.18a96921d97910a8 namespace: kserve-ci-e2e-test resourceVersion: "18683" uid: 991b4849-9a25-4406-8c2c-127ff2fb55d2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:22Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-e9425b-predictor-75f7ff4474-bpdjx namespace: kserve-ci-e2e-test resourceVersion: "18677" uid: 4a20173a-6851-4bc3-b559-cfa77ead9848 kind: Event lastTimestamp: "2026-04-24T21:45:22Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:45:22Z" name: isvc-init-fail-e9425b-predictor-75f7ff4474-bpdjx.18a969220b38a3c0 namespace: kserve-ci-e2e-test resourceVersion: "18695" uid: 7adb1833-04c8-419a-8a8a-a5f6508d214b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:45:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-e9425b-predictor-75f7ff4474-bpdjx namespace: kserve-ci-e2e-test resourceVersion: "18671" uid: 4a20173a-6851-4bc3-b559-cfa77ead9848 kind: Event lastTimestamp: "2026-04-24T21:45:29Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b2a48068363ac4e806775467ef2dc5b6c858faecf49ae1f8bafc4262d036f97e" already present on machine metadata: creationTimestamp: "2026-04-24T21:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:29Z" name: isvc-init-fail-e9425b-predictor-75f7ff4474-bpdjx.18a969220cb0986a namespace: kserve-ci-e2e-test resourceVersion: "18750" uid: fa1f7bae-76cf-4ce3-b803-fb45c294bc58 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:45:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-e9425b-predictor-75f7ff4474-bpdjx namespace: kserve-ci-e2e-test resourceVersion: "18671" uid: 4a20173a-6851-4bc3-b559-cfa77ead9848 kind: Event lastTimestamp: "2026-04-24T21:45:29Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:29Z" name: isvc-init-fail-e9425b-predictor-75f7ff4474-bpdjx.18a969220f47bc08 namespace: kserve-ci-e2e-test resourceVersion: "18753" uid: 45625ee1-d499-4336-8bc1-25b159191f42 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:45:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-e9425b-predictor-75f7ff4474-bpdjx namespace: kserve-ci-e2e-test resourceVersion: "18671" uid: 4a20173a-6851-4bc3-b559-cfa77ead9848 kind: Event lastTimestamp: "2026-04-24T21:45:29Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:29Z" name: isvc-init-fail-e9425b-predictor-75f7ff4474-bpdjx.18a969220fb502dc namespace: kserve-ci-e2e-test resourceVersion: "18755" uid: 7529edfe-6360-4b95-984f-232b8fd80e50 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-e9425b-predictor-75f7ff4474-bpdjx namespace: kserve-ci-e2e-test resourceVersion: "18671" uid: 4a20173a-6851-4bc3-b559-cfa77ead9848 kind: Event lastTimestamp: "2026-04-24T21:45:31Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-04-24T21:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:31Z" name: isvc-init-fail-e9425b-predictor-75f7ff4474-bpdjx.18a969241feba89e namespace: kserve-ci-e2e-test resourceVersion: "18797" uid: cac13d03-64c1-4495-b19d-0ba6cb7905ba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-e9425b-predictor-75f7ff4474 namespace: kserve-ci-e2e-test resourceVersion: "18660" uid: 83d8f292-08f8-4afa-9155-9fc55ff577d7 kind: Event lastTimestamp: "2026-04-24T21:45:21Z" message: 'Created pod: isvc-init-fail-e9425b-predictor-75f7ff4474-bpdjx' metadata: creationTimestamp: "2026-04-24T21:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:21Z" name: isvc-init-fail-e9425b-predictor-75f7ff4474.18a96921cd6b9711 namespace: kserve-ci-e2e-test resourceVersion: "18670" uid: eb3b1364-43fb-40bd-8b35-ad7e3952877a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-e9425b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18657" uid: 175f1b23-a3ad-4cc4-895b-f74632f02b9b kind: Event lastTimestamp: "2026-04-24T21:45:21Z" message: Scaled up replica set isvc-init-fail-e9425b-predictor-75f7ff4474 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:21Z" name: isvc-init-fail-e9425b-predictor.18a96921ca3b3753 namespace: kserve-ci-e2e-test resourceVersion: "18662" uid: a17283f6-3100-4866-8692-ccb01f3724a9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T21:45:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-e9425b namespace: kserve-ci-e2e-test resourceVersion: "18633" uid: bf0730ee-b1a0-4979-9f91-3d81b13b37c2 kind: Event lastTimestamp: "2026-04-24T21:45:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:29Z" name: isvc-init-fail-e9425b.18a96921c9cbc610 namespace: kserve-ci-e2e-test resourceVersion: "18758" uid: 27d43478-3eae-4e1d-a99f-7b997a09667e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-e9425b namespace: kserve-ci-e2e-test resourceVersion: "18633" uid: bf0730ee-b1a0-4979-9f91-3d81b13b37c2 kind: Event lastTimestamp: "2026-04-24T21:45:21Z" message: 'Failed to update status for InferenceService "isvc-init-fail-e9425b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-e9425b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:21Z" name: isvc-init-fail-e9425b.18a96921ca669855 namespace: kserve-ci-e2e-test resourceVersion: "18663" uid: 8d89d8d5-b859-41bf-862d-5e14a1816e69 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-e9425b namespace: kserve-ci-e2e-test resourceVersion: "18633" uid: bf0730ee-b1a0-4979-9f91-3d81b13b37c2 kind: Event lastTimestamp: "2026-04-24T21:45:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-e9425b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:21Z" name: isvc-init-fail-e9425b.18a96921ca66e0f1 namespace: kserve-ci-e2e-test resourceVersion: "18665" uid: 805e4d87-e226-495d-a951-3cb456c04f36 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:39:48.031680Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv namespace: kserve-ci-e2e-test resourceVersion: "15896" uid: 536557ba-cfa7-4d51-8905-09b279dcd213 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-24T21:39:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:39:48Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv.18a968d412c5de83 namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: 3157437f-14b3-4998-b521-bb84965e0b3b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6f5f9cfdf5-qjgtv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:48Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv namespace: kserve-ci-e2e-test resourceVersion: "15907" uid: 536557ba-cfa7-4d51-8905-09b279dcd213 kind: Event lastTimestamp: "2026-04-24T21:39:48Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:39:48Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv.18a968d42bcc7900 namespace: kserve-ci-e2e-test resourceVersion: "15918" uid: 83840167-3f70-458c-ab72-43cc03ad1e47 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 536557ba-cfa7-4d51-8905-09b279dcd213 kind: Event lastTimestamp: "2026-04-24T21:39:48Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b2a48068363ac4e806775467ef2dc5b6c858faecf49ae1f8bafc4262d036f97e" already present on machine metadata: creationTimestamp: "2026-04-24T21:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:48Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv.18a968d42da242a2 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 42a2d484-b4eb-4caa-8301-0796cfe80da1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 536557ba-cfa7-4d51-8905-09b279dcd213 kind: Event lastTimestamp: "2026-04-24T21:39:48Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:48Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv.18a968d4317bd75f namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 073ef23f-c62b-4642-9f13-4c2cd6ca1a33 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 536557ba-cfa7-4d51-8905-09b279dcd213 kind: Event lastTimestamp: "2026-04-24T21:39:48Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:48Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv.18a968d431eb587a namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: c9cf130d-4202-40c4-9f61-90a11d62b75d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 536557ba-cfa7-4d51-8905-09b279dcd213 kind: Event lastTimestamp: "2026-04-24T21:39:52Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:39:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:52Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv.18a968d50bedf13b namespace: kserve-ci-e2e-test resourceVersion: "15956" uid: aeae8924-eaed-4b46-a33f-27072bcf9c33 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 536557ba-cfa7-4d51-8905-09b279dcd213 kind: Event lastTimestamp: "2026-04-24T21:39:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:39:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:52Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv.18a968d50ed9aa7c namespace: kserve-ci-e2e-test resourceVersion: "15958" uid: 73791d3c-bafb-4e97-b453-525c79902078 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 536557ba-cfa7-4d51-8905-09b279dcd213 kind: Event lastTimestamp: "2026-04-24T21:39:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:39:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:52Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv.18a968d50f4d0419 namespace: kserve-ci-e2e-test resourceVersion: "15959" uid: 30744228-df53-42a6-89f0-a56842d7b660 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 536557ba-cfa7-4d51-8905-09b279dcd213 kind: Event lastTimestamp: "2026-04-24T21:39:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:39:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:52Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv.18a968d50f5992e1 namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 0884e5e5-f8fc-4045-8e0e-53b221364a75 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 536557ba-cfa7-4d51-8905-09b279dcd213 kind: Event lastTimestamp: "2026-04-24T21:39:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:39:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:52Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv.18a968d51284710e namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: 920f8222-2a85-44aa-82b4-53fc1ffa568b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 536557ba-cfa7-4d51-8905-09b279dcd213 kind: Event lastTimestamp: "2026-04-24T21:39:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:39:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:52Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv.18a968d512f66457 namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: f7b53eda-1b75-4c89-ab2f-ce552107cccd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 536557ba-cfa7-4d51-8905-09b279dcd213 kind: Event lastTimestamp: "2026-04-24T21:39:52Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:53da750273d8061e6014e5107f86ea181415ead6af42ce42cb4b4c967b3ef0a5" already present on machine metadata: creationTimestamp: "2026-04-24T21:39:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:52Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv.18a968d51301c10b namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 11af1033-6479-4170-95d0-0d11bd9c5331 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 536557ba-cfa7-4d51-8905-09b279dcd213 kind: Event lastTimestamp: "2026-04-24T21:39:52Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-24T21:39:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:52Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv.18a968d515eb35e6 namespace: kserve-ci-e2e-test resourceVersion: "15967" uid: 4022003d-a39b-4b0a-abd2-bc6516645025 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 536557ba-cfa7-4d51-8905-09b279dcd213 kind: Event lastTimestamp: "2026-04-24T21:39:52Z" message: Started container agent metadata: creationTimestamp: "2026-04-24T21:39:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:52Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv.18a968d516589354 namespace: kserve-ci-e2e-test resourceVersion: "15968" uid: b576868e-6a44-41bd-b19e-656e8a560f3d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-24T21:39:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 536557ba-cfa7-4d51-8905-09b279dcd213 kind: Event lastTimestamp: "2026-04-24T21:41:30Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:39:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:30Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv.18a968d547d9e5ed namespace: kserve-ci-e2e-test resourceVersion: "16806" uid: 21986115-e723-4bd7-90de-db1d42fd2760 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T21:39:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 536557ba-cfa7-4d51-8905-09b279dcd213 kind: Event lastTimestamp: "2026-04-24T21:41:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:39:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:30Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv.18a968d583968285 namespace: kserve-ci-e2e-test resourceVersion: "16807" uid: aeca8b0c-97fd-405c-9e47-e5e18a4ad6b7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 536557ba-cfa7-4d51-8905-09b279dcd213 kind: Event lastTimestamp: "2026-04-24T21:41:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:41:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:13Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv.18a968e7e6d2a546 namespace: kserve-ci-e2e-test resourceVersion: "16621" uid: d0de1da1-0d00-491c-b869-61c6e358d22c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 536557ba-cfa7-4d51-8905-09b279dcd213 kind: Event lastTimestamp: "2026-04-24T21:41:13Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-24T21:41:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:13Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv.18a968e7e6d30f59 namespace: kserve-ci-e2e-test resourceVersion: "16622" uid: 773b8dea-b61c-4ac9-92b3-e4542925d52d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 536557ba-cfa7-4d51-8905-09b279dcd213 kind: Event lastTimestamp: "2026-04-24T21:41:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:41:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:13Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv.18a968e7e6d3557e namespace: kserve-ci-e2e-test resourceVersion: "16624" uid: a874ef4d-31ec-4b18-aa62-a8a1a2a763ac reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:41:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 536557ba-cfa7-4d51-8905-09b279dcd213 kind: Event lastTimestamp: "2026-04-24T21:41:30Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp 10.133.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:41:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:30Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv.18a968e85fbdcd85 namespace: kserve-ci-e2e-test resourceVersion: "16805" uid: ddd39001-47ed-4b9a-9858-91fff3393993 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-9a20e-predictor-5976c8f66 namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: ca3defa2-6f1b-4bd2-bbf5-0e02109f2e83 kind: Event lastTimestamp: "2026-04-24T21:39:48Z" message: 'Created pod: isvc-logger-raw-9a20e-predictor-5976c8f66-dfrcv' metadata: creationTimestamp: "2026-04-24T21:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:48Z" name: isvc-logger-raw-9a20e-predictor-5976c8f66.18a968d411c9c10d namespace: kserve-ci-e2e-test resourceVersion: "15900" uid: 0a828be1-a248-449c-b9ad-09dac23b6ae3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-9a20e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15885" uid: c0f7368d-7a3b-43f6-bfa6-3f603e80aad8 kind: Event lastTimestamp: "2026-04-24T21:39:47Z" message: Scaled up replica set isvc-logger-raw-9a20e-predictor-5976c8f66 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:47Z" name: isvc-logger-raw-9a20e-predictor.18a968d40c9eb642 namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: fda39a6b-66ff-4d78-b9ff-fa72ab8cf84c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:40:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-9a20e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 6d281a71-a8c4-4e9c-b8de-30d869f9937f kind: Event lastTimestamp: "2026-04-24T21:40:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:32Z" name: isvc-logger-raw-9a20e-predictor.18a968d78b15c3b5 namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: c1eeec68-8078-421f-922e-a8209c33dff3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:40:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-9a20e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 6d281a71-a8c4-4e9c-b8de-30d869f9937f kind: Event lastTimestamp: "2026-04-24T21:40:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:32Z" name: isvc-logger-raw-9a20e-predictor.18a968d78b8314ff namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 049aaa07-6acd-416e-8a21-39361c6a7f5c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:40:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-9a20e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 6d281a71-a8c4-4e9c-b8de-30d869f9937f kind: Event lastTimestamp: "2026-04-24T21:41:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:03Z" name: isvc-logger-raw-9a20e-predictor.18a968e20876fb40 namespace: kserve-ci-e2e-test resourceVersion: "16500" uid: 339d0dd8-a565-45fb-80ad-b239e6db34b4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:40:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-9a20e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 6d281a71-a8c4-4e9c-b8de-30d869f9937f kind: Event lastTimestamp: "2026-04-24T21:41:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:03Z" name: isvc-logger-raw-9a20e-predictor.18a968e208fc887a namespace: kserve-ci-e2e-test resourceVersion: "16501" uid: 6db0ef0e-de9e-40f1-89f1-1781eaac3141 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T21:39:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-9a20e namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: b25e64b8-7767-44f7-9ffc-d71ca68553a6 kind: Event lastTimestamp: "2026-04-24T21:41:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:41:00Z" name: isvc-logger-raw-9a20e.18a968d40c7d8883 namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 4d8c3de3-8556-4676-b0d0-5ff3e0a4243b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:39:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-9a20e namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: b25e64b8-7767-44f7-9ffc-d71ca68553a6 kind: Event lastTimestamp: "2026-04-24T21:39:48Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-9a20e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-9a20e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:48Z" name: isvc-logger-raw-9a20e.18a968d40cf1fab3 namespace: kserve-ci-e2e-test resourceVersion: "15912" uid: 259897c7-13db-4aa0-87c7-e6f314c89826 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-9a20e namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: b25e64b8-7767-44f7-9ffc-d71ca68553a6 kind: Event lastTimestamp: "2026-04-24T21:39:48Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-9a20e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:48Z" name: isvc-logger-raw-9a20e.18a968d416933f8a namespace: kserve-ci-e2e-test resourceVersion: "15913" uid: 19ad7f88-4ef4-4cf4-9598-617b50b66569 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-9a20e namespace: kserve-ci-e2e-test resourceVersion: "16476" uid: b25e64b8-7767-44f7-9ffc-d71ca68553a6 kind: Event lastTimestamp: "2026-04-24T21:41:00Z" message: InferenceService [isvc-logger-raw-9a20e] is Ready metadata: creationTimestamp: "2026-04-24T21:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:41:00Z" name: isvc-logger-raw-9a20e.18a968e4e9448587 namespace: kserve-ci-e2e-test resourceVersion: "16479" uid: 543c163f-5065-4497-a0d0-b9bafd030642 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:43:43.491725Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-ab1d4b-predictor-669544d645-n5l22 namespace: kserve-ci-e2e-test resourceVersion: "17784" uid: 19f3e54f-d5d6-4a7a-b4d9-b86fea03954e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-ab1d4b-predictor-669544d645-n5l22 to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-24T21:43:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:43:43Z" name: isvc-primary-ab1d4b-predictor-669544d645-n5l22.18a9690ae5491de1 namespace: kserve-ci-e2e-test resourceVersion: "17788" uid: de42cdc6-e2cf-435b-83ff-3ed24abe2290 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6f5f9cfdf5-qjgtv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-ab1d4b-predictor-669544d645-n5l22 namespace: kserve-ci-e2e-test resourceVersion: "17786" uid: 19f3e54f-d5d6-4a7a-b4d9-b86fea03954e kind: Event lastTimestamp: "2026-04-24T21:43:43Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-primary-ab1d4b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:43Z" name: isvc-primary-ab1d4b-predictor-669544d645-n5l22.18a9690af3c55469 namespace: kserve-ci-e2e-test resourceVersion: "17796" uid: 6a9115be-0035-4c6c-ba41-0cb165ca50d2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:44Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-ab1d4b-predictor-669544d645-n5l22 namespace: kserve-ci-e2e-test resourceVersion: "17792" uid: 19f3e54f-d5d6-4a7a-b4d9-b86fea03954e kind: Event lastTimestamp: "2026-04-24T21:43:44Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:43:44Z" name: isvc-primary-ab1d4b-predictor-669544d645-n5l22.18a9690b227fdf55 namespace: kserve-ci-e2e-test resourceVersion: "17807" uid: d567752b-ba43-49c4-87a1-54829dfbeab9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-ab1d4b-predictor-669544d645-n5l22 namespace: kserve-ci-e2e-test resourceVersion: "17786" uid: 19f3e54f-d5d6-4a7a-b4d9-b86fea03954e kind: Event lastTimestamp: "2026-04-24T21:43:44Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b2a48068363ac4e806775467ef2dc5b6c858faecf49ae1f8bafc4262d036f97e" already present on machine metadata: creationTimestamp: "2026-04-24T21:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:44Z" name: isvc-primary-ab1d4b-predictor-669544d645-n5l22.18a9690b23e46c0e namespace: kserve-ci-e2e-test resourceVersion: "17809" uid: 7a1ee3af-dec5-489d-93b7-52151dd2ba2c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-ab1d4b-predictor-669544d645-n5l22 namespace: kserve-ci-e2e-test resourceVersion: "17786" uid: 19f3e54f-d5d6-4a7a-b4d9-b86fea03954e kind: Event lastTimestamp: "2026-04-24T21:43:44Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:44Z" name: isvc-primary-ab1d4b-predictor-669544d645-n5l22.18a9690b2786c87a namespace: kserve-ci-e2e-test resourceVersion: "17810" uid: bbc3cfa9-c599-4ce0-80da-266bbc05e710 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-ab1d4b-predictor-669544d645-n5l22 namespace: kserve-ci-e2e-test resourceVersion: "17786" uid: 19f3e54f-d5d6-4a7a-b4d9-b86fea03954e kind: Event lastTimestamp: "2026-04-24T21:43:44Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:44Z" name: isvc-primary-ab1d4b-predictor-669544d645-n5l22.18a9690b27f86736 namespace: kserve-ci-e2e-test resourceVersion: "17811" uid: 26fa7a71-e923-4594-b2fa-0402f6888c13 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-ab1d4b-predictor-669544d645-n5l22 namespace: kserve-ci-e2e-test resourceVersion: "17786" uid: 19f3e54f-d5d6-4a7a-b4d9-b86fea03954e kind: Event lastTimestamp: "2026-04-24T21:43:48Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:43:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:48Z" name: isvc-primary-ab1d4b-predictor-669544d645-n5l22.18a9690c28406879 namespace: kserve-ci-e2e-test resourceVersion: "17860" uid: eb212353-11f0-4f57-b7d2-21044dd2e58d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-ab1d4b-predictor-669544d645-n5l22 namespace: kserve-ci-e2e-test resourceVersion: "17786" uid: 19f3e54f-d5d6-4a7a-b4d9-b86fea03954e kind: Event lastTimestamp: "2026-04-24T21:43:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:43:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:48Z" name: isvc-primary-ab1d4b-predictor-669544d645-n5l22.18a9690c2bdb2f5f namespace: kserve-ci-e2e-test resourceVersion: "17863" uid: 4cc9ce48-eac4-41d4-8542-666dec8371fc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-ab1d4b-predictor-669544d645-n5l22 namespace: kserve-ci-e2e-test resourceVersion: "17786" uid: 19f3e54f-d5d6-4a7a-b4d9-b86fea03954e kind: Event lastTimestamp: "2026-04-24T21:43:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:43:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:48Z" name: isvc-primary-ab1d4b-predictor-669544d645-n5l22.18a9690c2c542794 namespace: kserve-ci-e2e-test resourceVersion: "17866" uid: a6bacb29-8d64-48ac-81ef-bd2b0008898c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-ab1d4b-predictor-669544d645-n5l22 namespace: kserve-ci-e2e-test resourceVersion: "17786" uid: 19f3e54f-d5d6-4a7a-b4d9-b86fea03954e kind: Event lastTimestamp: "2026-04-24T21:43:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:43:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:48Z" name: isvc-primary-ab1d4b-predictor-669544d645-n5l22.18a9690c2c603e37 namespace: kserve-ci-e2e-test resourceVersion: "17867" uid: ac9d3d5f-74f6-456f-a4d6-ecaaada9b8dd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-ab1d4b-predictor-669544d645-n5l22 namespace: kserve-ci-e2e-test resourceVersion: "17786" uid: 19f3e54f-d5d6-4a7a-b4d9-b86fea03954e kind: Event lastTimestamp: "2026-04-24T21:43:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:49Z" name: isvc-primary-ab1d4b-predictor-669544d645-n5l22.18a9690c2f0052ed namespace: kserve-ci-e2e-test resourceVersion: "17868" uid: 21a57ec9-2f9b-4339-bdf5-3ae9a0ef0663 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-ab1d4b-predictor-669544d645-n5l22 namespace: kserve-ci-e2e-test resourceVersion: "17786" uid: 19f3e54f-d5d6-4a7a-b4d9-b86fea03954e kind: Event lastTimestamp: "2026-04-24T21:43:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:49Z" name: isvc-primary-ab1d4b-predictor-669544d645-n5l22.18a9690c2f746e26 namespace: kserve-ci-e2e-test resourceVersion: "17870" uid: 06f605c9-d8ec-481d-bdd6-a897fd6eb822 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T21:43:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-ab1d4b-predictor-669544d645-n5l22 namespace: kserve-ci-e2e-test resourceVersion: "17786" uid: 19f3e54f-d5d6-4a7a-b4d9-b86fea03954e kind: Event lastTimestamp: "2026-04-24T21:44:46Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:43:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:44:46Z" name: isvc-primary-ab1d4b-predictor-669544d645-n5l22.18a9690c9fef6d95 namespace: kserve-ci-e2e-test resourceVersion: "18288" uid: 24b152cd-2df5-4209-ba55-73a99fba5fa0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-ab1d4b-predictor-669544d645-n5l22 namespace: kserve-ci-e2e-test resourceVersion: "17786" uid: 19f3e54f-d5d6-4a7a-b4d9-b86fea03954e kind: Event lastTimestamp: "2026-04-24T21:45:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:21Z" name: isvc-primary-ab1d4b-predictor-669544d645-n5l22.18a96921c26c9964 namespace: kserve-ci-e2e-test resourceVersion: "18631" uid: 29d9eb89-5197-44b6-8992-5826db0810fc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-ab1d4b-predictor-669544d645-n5l22 namespace: kserve-ci-e2e-test resourceVersion: "17786" uid: 19f3e54f-d5d6-4a7a-b4d9-b86fea03954e kind: Event lastTimestamp: "2026-04-24T21:45:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:21Z" name: isvc-primary-ab1d4b-predictor-669544d645-n5l22.18a96921c26d299e namespace: kserve-ci-e2e-test resourceVersion: "18632" uid: e8d3b935-b9f9-4006-8c12-e616a38f1a2e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-ab1d4b-predictor-669544d645-n5l22 namespace: kserve-ci-e2e-test resourceVersion: "17786" uid: 19f3e54f-d5d6-4a7a-b4d9-b86fea03954e kind: Event lastTimestamp: "2026-04-24T21:45:21Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:21Z" name: isvc-primary-ab1d4b-predictor-669544d645-n5l22.18a96921d00dd6f8 namespace: kserve-ci-e2e-test resourceVersion: "18678" uid: 44a5330b-238f-4ae6-8a1e-c96081d0c5f7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-ab1d4b-predictor-669544d645 namespace: kserve-ci-e2e-test resourceVersion: "17775" uid: 3faf8dce-fa9b-4845-9055-694f84fc9b77 kind: Event lastTimestamp: "2026-04-24T21:43:43Z" message: 'Created pod: isvc-primary-ab1d4b-predictor-669544d645-n5l22' metadata: creationTimestamp: "2026-04-24T21:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:43:43Z" name: isvc-primary-ab1d4b-predictor-669544d645.18a9690ae4d89c18 namespace: kserve-ci-e2e-test resourceVersion: "17785" uid: b584ac6f-51c8-4eb3-84c8-6f46e9d1ae00 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-ab1d4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17772" uid: 77d044d0-5c41-4c02-b3fd-744fe1af641e kind: Event lastTimestamp: "2026-04-24T21:43:43Z" message: Scaled up replica set isvc-primary-ab1d4b-predictor-669544d645 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:43:43Z" name: isvc-primary-ab1d4b-predictor.18a9690ae070aad8 namespace: kserve-ci-e2e-test resourceVersion: "17776" uid: e4819483-0dfb-44fd-8cc6-d98dd365807a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:43:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-ab1d4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17774" uid: c0893b0e-f865-405c-a113-a33916377539 kind: Event lastTimestamp: "2026-04-24T21:44:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:44:28Z" name: isvc-primary-ab1d4b-predictor.18a9690e5fb04249 namespace: kserve-ci-e2e-test resourceVersion: "18158" uid: 6236c972-a4e2-4cb8-bdf6-bb6265fbd40e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:43:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-ab1d4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17774" uid: c0893b0e-f865-405c-a113-a33916377539 kind: Event lastTimestamp: "2026-04-24T21:44:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:44:28Z" name: isvc-primary-ab1d4b-predictor.18a9690e60209a59 namespace: kserve-ci-e2e-test resourceVersion: "18159" uid: 9009a2a6-a2cf-43f3-b257-3cf50c166a76 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:44:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-ab1d4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17945" uid: c0893b0e-f865-405c-a113-a33916377539 kind: Event lastTimestamp: "2026-04-24T21:45:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:44:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:13Z" name: isvc-primary-ab1d4b-predictor.18a96911ded420c9 namespace: kserve-ci-e2e-test resourceVersion: "18540" uid: 5f79f16c-95f3-4b63-974a-ab58f0446da9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:44:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-ab1d4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17945" uid: c0893b0e-f865-405c-a113-a33916377539 kind: Event lastTimestamp: "2026-04-24T21:45:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:44:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:13Z" name: isvc-primary-ab1d4b-predictor.18a96911df3f5cdd namespace: kserve-ci-e2e-test resourceVersion: "18541" uid: f87eea75-e948-442c-b1a1-08755cc11bab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-24T21:43:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-ab1d4b namespace: kserve-ci-e2e-test resourceVersion: "17745" uid: da6fdc93-d548-4a65-99eb-19b50071e2ca kind: Event lastTimestamp: "2026-04-24T21:44:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:44:57Z" name: isvc-primary-ab1d4b.18a9690ae07ae328 namespace: kserve-ci-e2e-test resourceVersion: "18363" uid: 214a5cea-b8f8-454d-afe8-8d6edd7bfd9f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-ab1d4b namespace: kserve-ci-e2e-test resourceVersion: "17745" uid: da6fdc93-d548-4a65-99eb-19b50071e2ca kind: Event lastTimestamp: "2026-04-24T21:43:43Z" message: 'Failed to update status for InferenceService "isvc-primary-ab1d4b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-ab1d4b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:43:43Z" name: isvc-primary-ab1d4b.18a9690ae10464ad namespace: kserve-ci-e2e-test resourceVersion: "17779" uid: d17f6560-9dbb-43ba-a81e-f3865bbba2d7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-ab1d4b namespace: kserve-ci-e2e-test resourceVersion: "17745" uid: da6fdc93-d548-4a65-99eb-19b50071e2ca kind: Event lastTimestamp: "2026-04-24T21:43:43Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-ab1d4b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:43:43Z" name: isvc-primary-ab1d4b.18a9690ae104acaa namespace: kserve-ci-e2e-test resourceVersion: "17781" uid: 68b7a71f-f20f-49da-9b60-7c92d6031c05 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:44:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-ab1d4b namespace: kserve-ci-e2e-test resourceVersion: "18361" uid: da6fdc93-d548-4a65-99eb-19b50071e2ca kind: Event lastTimestamp: "2026-04-24T21:44:57Z" message: InferenceService [isvc-primary-ab1d4b] is Ready metadata: creationTimestamp: "2026-04-24T21:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:44:57Z" name: isvc-primary-ab1d4b.18a9691c04e31c1d namespace: kserve-ci-e2e-test resourceVersion: "18362" uid: 20bd868c-83cb-4509-a579-73cc2760d915 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:34:12.217009Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12810" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-24T21:34:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:34:12Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a96885e2a8af75 namespace: kserve-ci-e2e-test resourceVersion: "12816" uid: cba2b446-e46f-4656-90b5-15bc5be0cf3e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6f5f9cfdf5-qjgtv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:12Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12823" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:34:12Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:34:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:34:12Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a96885fb8bdba0 namespace: kserve-ci-e2e-test resourceVersion: "12832" uid: 0eb0282b-467c-4355-9838-ae49a38e42a6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:34:12Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:b2a48068363ac4e806775467ef2dc5b6c858faecf49ae1f8bafc4262d036f97e" metadata: creationTimestamp: "2026-04-24T21:34:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:12Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a96885fd559dad namespace: kserve-ci-e2e-test resourceVersion: "12834" uid: b055ecb2-6c96-4de1-b38d-0909c0cb86a4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:34:16Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:b2a48068363ac4e806775467ef2dc5b6c858faecf49ae1f8bafc4262d036f97e" in 4.188s (4.188s including waiting). Image size: 299849138 bytes.' metadata: creationTimestamp: "2026-04-24T21:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:16Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a96886f6ff1cd8 namespace: kserve-ci-e2e-test resourceVersion: "12868" uid: 4ba5f5ee-865d-4d71-8c6f-220a77c4c76e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:34:16Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:16Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a96886f9c682af namespace: kserve-ci-e2e-test resourceVersion: "12871" uid: 5702a7ce-530a-4790-b0d1-c3a426f8f4ac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:34:16Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:16Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a96886fa25be74 namespace: kserve-ci-e2e-test resourceVersion: "12872" uid: 8f02c2f9-024c-4897-9d10-0fb7d5b28a45 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:34:21Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1449" metadata: creationTimestamp: "2026-04-24T21:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:21Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a96887f71c36ae namespace: kserve-ci-e2e-test resourceVersion: "12909" uid: f932da0e-ded0-43fa-957b-7a1335cb6d59 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:34:34Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1449" in 13.281s (13.281s including waiting). Image size: 1560926130 bytes.' metadata: creationTimestamp: "2026-04-24T21:34:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:34Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a9688b0ec0e497 namespace: kserve-ci-e2e-test resourceVersion: "13009" uid: bb11ef53-e5fb-40a8-a1e7-2c75fadfbbfa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:34:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:34:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:34Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a9688b11820a3e namespace: kserve-ci-e2e-test resourceVersion: "13011" uid: a1cfa92b-d9f3-4c47-b894-444973ee6137 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:34:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:34:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:34Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a9688b11e21a53 namespace: kserve-ci-e2e-test resourceVersion: "13012" uid: 9e979294-8d68-4540-a46f-3670103d4ce1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:34:34Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T21:34:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:34Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a9688b11ed648c namespace: kserve-ci-e2e-test resourceVersion: "13013" uid: 7ac4e426-893d-4f3f-98d9-7a91d3e7004f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:34:36Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.296s (2.296s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T21:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:36Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a9688b9ad04970 namespace: kserve-ci-e2e-test resourceVersion: "13027" uid: 970cdf6a-f182-4f26-a862-8df84c516141 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:34:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:36Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a9688b9d8fad0b namespace: kserve-ci-e2e-test resourceVersion: "13028" uid: 83427aa9-15ea-4b81-a1cb-03815c627d96 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:34:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:36Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a9688b9df350f7 namespace: kserve-ci-e2e-test resourceVersion: "13029" uid: 840fab7b-2919-4090-acc8-9046e7d29bdb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:34:36Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:53da750273d8061e6014e5107f86ea181415ead6af42ce42cb4b4c967b3ef0a5" metadata: creationTimestamp: "2026-04-24T21:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:36Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a9688b9dff88ce namespace: kserve-ci-e2e-test resourceVersion: "13030" uid: 0ec24e45-854d-4d3c-86ab-7a9d7060b3d0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:34:39Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:53da750273d8061e6014e5107f86ea181415ead6af42ce42cb4b4c967b3ef0a5" in 2.49s (2.49s including waiting). Image size: 238035033 bytes.' metadata: creationTimestamp: "2026-04-24T21:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:39Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a9688c3272557d namespace: kserve-ci-e2e-test resourceVersion: "13046" uid: d327cf32-1915-46a4-8d89-9e5d296d8c75 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:34:39Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-24T21:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:39Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a9688c350685f1 namespace: kserve-ci-e2e-test resourceVersion: "13047" uid: 31a7e053-6c03-478a-a657-959b50e6848d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:34:39Z" message: Started container agent metadata: creationTimestamp: "2026-04-24T21:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:39Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a9688c35633f19 namespace: kserve-ci-e2e-test resourceVersion: "13048" uid: 20f2e6a2-6fac-4b03-84b0-676aab72ce5a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T21:34:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:36:12Z" message: 'Readiness probe failed: dial tcp 10.133.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:12Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a9688c67d37af2 namespace: kserve-ci-e2e-test resourceVersion: "13867" uid: dc411180-5a8d-4891-bf52-dabad3e41723 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T21:34:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:36:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:12Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a9688c67dd166f namespace: kserve-ci-e2e-test resourceVersion: "13870" uid: a2db3717-260e-4875-8758-91d561e76e55 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:35:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:57Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a9689e5575c1f9 namespace: kserve-ci-e2e-test resourceVersion: "13648" uid: cee25b32-67cb-405d-8313-7263369f575a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:35:57Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:57Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a9689e5576e27a namespace: kserve-ci-e2e-test resourceVersion: "13652" uid: f8140b18-7455-4695-9c50-d900ea3dc0d0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:35:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:57Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a9689e5578156d namespace: kserve-ci-e2e-test resourceVersion: "13655" uid: 589553e8-c823-4168-9a34-c3f66d898399 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:36:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: 4650e380-d5f3-407b-8763-5e988c00f004 kind: Event lastTimestamp: "2026-04-24T21:36:21Z" message: 'Readiness probe failed: Get "https://10.133.0.20:8643/healthz": dial tcp 10.133.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:36:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:21Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl.18a9689f43eea5d1 namespace: kserve-ci-e2e-test resourceVersion: "13941" uid: a3c12af6-f62a-4a63-a8ef-9a32afc21ad1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786 namespace: kserve-ci-e2e-test resourceVersion: "12800" uid: 18ed6454-6b91-4b6a-80a2-06a8b8e0460b kind: Event lastTimestamp: "2026-04-24T21:34:12Z" message: 'Created pod: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786-666dl' metadata: creationTimestamp: "2026-04-24T21:34:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:12Z" name: isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786.18a96885e1eb8dc4 namespace: kserve-ci-e2e-test resourceVersion: "12812" uid: 86222dbb-d204-452a-8eb1-fda2d76b1257 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-2f47a-predictor namespace: kserve-ci-e2e-test resourceVersion: "12798" uid: 3b393c98-c9a3-4080-b74e-8549e2a444be kind: Event lastTimestamp: "2026-04-24T21:34:12Z" message: Scaled up replica set isvc-raw-sklearn-batcher-2f47a-predictor-5c664b4786 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:34:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:12Z" name: isvc-raw-sklearn-batcher-2f47a-predictor.18a96885dba8de58 namespace: kserve-ci-e2e-test resourceVersion: "12801" uid: e803d996-1ed0-4b54-93ed-6c1ef416e034 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:34:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-2f47a-predictor namespace: kserve-ci-e2e-test resourceVersion: "12809" uid: d35d9374-f558-4c0f-9e6f-7a6a2f5f2432 kind: Event lastTimestamp: "2026-04-24T21:35:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:12Z" name: isvc-raw-sklearn-batcher-2f47a-predictor.18a9688960c77302 namespace: kserve-ci-e2e-test resourceVersion: "13295" uid: 75d11201-a22a-4d73-9fa4-40fb455d7475 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:34:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-2f47a-predictor namespace: kserve-ci-e2e-test resourceVersion: "12809" uid: d35d9374-f558-4c0f-9e6f-7a6a2f5f2432 kind: Event lastTimestamp: "2026-04-24T21:35:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:12Z" name: isvc-raw-sklearn-batcher-2f47a-predictor.18a9688961365053 namespace: kserve-ci-e2e-test resourceVersion: "13296" uid: 4c371c60-07e3-4a3f-a686-30e49cd42e15 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:34:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-2f47a-predictor namespace: kserve-ci-e2e-test resourceVersion: "12960" uid: d35d9374-f558-4c0f-9e6f-7a6a2f5f2432 kind: Event lastTimestamp: "2026-04-24T21:35:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:34:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:42Z" name: isvc-raw-sklearn-batcher-2f47a-predictor.18a968905ef44fc5 namespace: kserve-ci-e2e-test resourceVersion: "13513" uid: b439954d-e7c8-4854-b974-cda2261a8162 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:34:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-2f47a-predictor namespace: kserve-ci-e2e-test resourceVersion: "12960" uid: d35d9374-f558-4c0f-9e6f-7a6a2f5f2432 kind: Event lastTimestamp: "2026-04-24T21:35:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:34:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:42Z" name: isvc-raw-sklearn-batcher-2f47a-predictor.18a968905f5cc381 namespace: kserve-ci-e2e-test resourceVersion: "13514" uid: 63cf1d0e-1e6e-4821-bb32-b251122ee7ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T21:34:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-2f47a namespace: kserve-ci-e2e-test resourceVersion: "12787" uid: 289ed630-e5c3-4c0e-b659-018ad3e77635 kind: Event lastTimestamp: "2026-04-24T21:35:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:34:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:35:42Z" name: isvc-raw-sklearn-batcher-2f47a.18a96885e182cab7 namespace: kserve-ci-e2e-test resourceVersion: "13518" uid: b1b7ab30-6cc2-4c8e-8fdb-61cee1a43356 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-2f47a namespace: kserve-ci-e2e-test resourceVersion: "12787" uid: 289ed630-e5c3-4c0e-b659-018ad3e77635 kind: Event lastTimestamp: "2026-04-24T21:34:12Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-2f47a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-2f47a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:34:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:34:12Z" name: isvc-raw-sklearn-batcher-2f47a.18a96885e20cab63 namespace: kserve-ci-e2e-test resourceVersion: "12815" uid: 5f4c00f7-dd16-4ce0-a516-8e8da05d49da reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-2f47a namespace: kserve-ci-e2e-test resourceVersion: "12787" uid: 289ed630-e5c3-4c0e-b659-018ad3e77635 kind: Event lastTimestamp: "2026-04-24T21:34:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-2f47a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:34:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:34:12Z" name: isvc-raw-sklearn-batcher-2f47a.18a96885e20cfd5a namespace: kserve-ci-e2e-test resourceVersion: "12817" uid: 147bb994-6abc-4356-b9b9-fe5d5374c900 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-2f47a namespace: kserve-ci-e2e-test resourceVersion: "13515" uid: 289ed630-e5c3-4c0e-b659-018ad3e77635 kind: Event lastTimestamp: "2026-04-24T21:35:42Z" message: InferenceService [isvc-raw-sklearn-batcher-2f47a] is Ready metadata: creationTimestamp: "2026-04-24T21:35:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:35:42Z" name: isvc-raw-sklearn-batcher-2f47a.18a9689adf956f6e namespace: kserve-ci-e2e-test resourceVersion: "13517" uid: 26baa89c-5913-4195-b506-e4a850e07044 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:45:03.654041Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-ab1d4b-predictor-5c9b76987d-qd29f namespace: kserve-ci-e2e-test resourceVersion: "18433" uid: a04a75f1-4fff-4cef-b530-d1dba24e65e1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-ab1d4b-predictor-5c9b76987d-qd29f to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-24T21:45:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:45:03Z" name: isvc-secondary-ab1d4b-predictor-5c9b76987d-qd29f.18a9691d8f54fc53 namespace: kserve-ci-e2e-test resourceVersion: "18437" uid: f6845475-8c93-4ac6-952e-5b24f3f7c4c3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6f5f9cfdf5-qjgtv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:03Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-ab1d4b-predictor-5c9b76987d-qd29f namespace: kserve-ci-e2e-test resourceVersion: "18435" uid: a04a75f1-4fff-4cef-b530-d1dba24e65e1 kind: Event lastTimestamp: "2026-04-24T21:45:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-secondary-ab1d4b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:03Z" name: isvc-secondary-ab1d4b-predictor-5c9b76987d-qd29f.18a9691d9a572226 namespace: kserve-ci-e2e-test resourceVersion: "18445" uid: 3a91de1c-45a3-43b3-badb-69c51802115c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-ab1d4b-predictor-5c9b76987d-qd29f namespace: kserve-ci-e2e-test resourceVersion: "18442" uid: a04a75f1-4fff-4cef-b530-d1dba24e65e1 kind: Event lastTimestamp: "2026-04-24T21:45:04Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:45:04Z" name: isvc-secondary-ab1d4b-predictor-5c9b76987d-qd29f.18a9691dcc17f03c namespace: kserve-ci-e2e-test resourceVersion: "18454" uid: b9fce12c-0c0a-4d01-852f-4a6183ebead3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-ab1d4b-predictor-5c9b76987d-qd29f namespace: kserve-ci-e2e-test resourceVersion: "18435" uid: a04a75f1-4fff-4cef-b530-d1dba24e65e1 kind: Event lastTimestamp: "2026-04-24T21:45:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b2a48068363ac4e806775467ef2dc5b6c858faecf49ae1f8bafc4262d036f97e" already present on machine metadata: creationTimestamp: "2026-04-24T21:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:11Z" name: isvc-secondary-ab1d4b-predictor-5c9b76987d-qd29f.18a9691dcd8ab914 namespace: kserve-ci-e2e-test resourceVersion: "18515" uid: 65a18d82-c786-426a-9f94-d9b2a1130177 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-ab1d4b-predictor-5c9b76987d-qd29f namespace: kserve-ci-e2e-test resourceVersion: "18435" uid: a04a75f1-4fff-4cef-b530-d1dba24e65e1 kind: Event lastTimestamp: "2026-04-24T21:45:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:11Z" name: isvc-secondary-ab1d4b-predictor-5c9b76987d-qd29f.18a9691dd09bb573 namespace: kserve-ci-e2e-test resourceVersion: "18517" uid: a5e1ec4c-9975-4a82-a948-8c0534f5f5a1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-ab1d4b-predictor-5c9b76987d-qd29f namespace: kserve-ci-e2e-test resourceVersion: "18435" uid: a04a75f1-4fff-4cef-b530-d1dba24e65e1 kind: Event lastTimestamp: "2026-04-24T21:45:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:11Z" name: isvc-secondary-ab1d4b-predictor-5c9b76987d-qd29f.18a9691dd1108239 namespace: kserve-ci-e2e-test resourceVersion: "18518" uid: c2a597c1-342a-4d39-a1f4-b35b2ee1b6af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-ab1d4b-predictor-5c9b76987d-qd29f namespace: kserve-ci-e2e-test resourceVersion: "18435" uid: a04a75f1-4fff-4cef-b530-d1dba24e65e1 kind: Event lastTimestamp: "2026-04-24T21:45:16Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-ab1d4b-predictor-5c9b76987d-qd29f_kserve-ci-e2e-test(a04a75f1-4fff-4cef-b530-d1dba24e65e1) metadata: creationTimestamp: "2026-04-24T21:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:16Z" name: isvc-secondary-ab1d4b-predictor-5c9b76987d-qd29f.18a969207a19678d namespace: kserve-ci-e2e-test resourceVersion: "18558" uid: 57a6124b-6788-4fd0-aeb2-4f52993af374 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-ab1d4b-predictor-5c9b76987d namespace: kserve-ci-e2e-test resourceVersion: "18424" uid: 850bad94-6254-4b46-a476-ac559f6d3f25 kind: Event lastTimestamp: "2026-04-24T21:45:03Z" message: 'Created pod: isvc-secondary-ab1d4b-predictor-5c9b76987d-qd29f' metadata: creationTimestamp: "2026-04-24T21:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:03Z" name: isvc-secondary-ab1d4b-predictor-5c9b76987d.18a9691d8eaeedc0 namespace: kserve-ci-e2e-test resourceVersion: "18434" uid: 9ba8c980-b786-480e-a2fb-1d881a63f344 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-ab1d4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18422" uid: 43b7d66e-61ec-44a3-ab7f-ab16da75cced kind: Event lastTimestamp: "2026-04-24T21:45:03Z" message: Scaled up replica set isvc-secondary-ab1d4b-predictor-5c9b76987d from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:03Z" name: isvc-secondary-ab1d4b-predictor.18a9691d8b533398 namespace: kserve-ci-e2e-test resourceVersion: "18426" uid: 1c41422d-70fd-4926-99b4-8d9ef9d19706 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-ab1d4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18423" uid: e760c89c-bb5c-4866-927c-1fdac73ecee2 kind: Event lastTimestamp: "2026-04-24T21:45:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:45:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:18Z" name: isvc-secondary-ab1d4b-predictor.18a969210a30019c namespace: kserve-ci-e2e-test resourceVersion: "18582" uid: 9b8c5237-fdf9-4c4f-b4f5-eb44dce602b3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-ab1d4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18423" uid: e760c89c-bb5c-4866-927c-1fdac73ecee2 kind: Event lastTimestamp: "2026-04-24T21:45:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:45:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:18Z" name: isvc-secondary-ab1d4b-predictor.18a969210a98d0e7 namespace: kserve-ci-e2e-test resourceVersion: "18584" uid: 9af49f05-5406-439f-a328-f4647be8c2ce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-24T21:45:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-ab1d4b namespace: kserve-ci-e2e-test resourceVersion: "18412" uid: 80b3b918-8f82-408e-82b5-4bf603f0dc0a kind: Event lastTimestamp: "2026-04-24T21:45:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:17Z" name: isvc-secondary-ab1d4b.18a9691d8b333568 namespace: kserve-ci-e2e-test resourceVersion: "18576" uid: 8b208562-aea8-46eb-91d2-6a7cea370a4c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-ab1d4b namespace: kserve-ci-e2e-test resourceVersion: "18412" uid: 80b3b918-8f82-408e-82b5-4bf603f0dc0a kind: Event lastTimestamp: "2026-04-24T21:45:03Z" message: 'Failed to update status for InferenceService "isvc-secondary-ab1d4b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-ab1d4b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:03Z" name: isvc-secondary-ab1d4b.18a9691d8bbaf3f3 namespace: kserve-ci-e2e-test resourceVersion: "18428" uid: b418f9d6-350f-47c3-a6ed-a3f0ccefb11e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-ab1d4b namespace: kserve-ci-e2e-test resourceVersion: "18412" uid: 80b3b918-8f82-408e-82b5-4bf603f0dc0a kind: Event lastTimestamp: "2026-04-24T21:45:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-ab1d4b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:03Z" name: isvc-secondary-ab1d4b.18a9691d8bbb3f6b namespace: kserve-ci-e2e-test resourceVersion: "18429" uid: b275c591-9a63-4820-9fd5-5031290e3f30 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:35:57.301268Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk namespace: kserve-ci-e2e-test resourceVersion: "13662" uid: 0a1cfccd-3d5e-4f76-a972-59d416230e95 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:35:57Z" name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk.18a9689e5a2b3c54 namespace: kserve-ci-e2e-test resourceVersion: "13670" uid: ad87dfdd-e1cb-43fb-85c1-cbf665314045 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6f5f9cfdf5-qjgtv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk namespace: kserve-ci-e2e-test resourceVersion: "13680" uid: 0a1cfccd-3d5e-4f76-a972-59d416230e95 kind: Event lastTimestamp: "2026-04-24T21:35:57Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:35:57Z" name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk.18a9689e72c75e04 namespace: kserve-ci-e2e-test resourceVersion: "13709" uid: 05b1263a-5558-4cdd-9bad-75f76b82ecdf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk namespace: kserve-ci-e2e-test resourceVersion: "13667" uid: 0a1cfccd-3d5e-4f76-a972-59d416230e95 kind: Event lastTimestamp: "2026-04-24T21:35:57Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b2a48068363ac4e806775467ef2dc5b6c858faecf49ae1f8bafc4262d036f97e" already present on machine metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:57Z" name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk.18a9689e74339e6a namespace: kserve-ci-e2e-test resourceVersion: "13711" uid: 7fd8012e-39be-4110-abc3-20139f5789b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk namespace: kserve-ci-e2e-test resourceVersion: "13667" uid: 0a1cfccd-3d5e-4f76-a972-59d416230e95 kind: Event lastTimestamp: "2026-04-24T21:35:57Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:57Z" name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk.18a9689e76871c54 namespace: kserve-ci-e2e-test resourceVersion: "13714" uid: ad751f14-4630-4422-bd6e-cd00342914bf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk namespace: kserve-ci-e2e-test resourceVersion: "13667" uid: 0a1cfccd-3d5e-4f76-a972-59d416230e95 kind: Event lastTimestamp: "2026-04-24T21:35:57Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:57Z" name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk.18a9689e76e5bf99 namespace: kserve-ci-e2e-test resourceVersion: "13715" uid: fa8dabff-cabf-40af-89e2-ef4b6972a36d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk namespace: kserve-ci-e2e-test resourceVersion: "13667" uid: 0a1cfccd-3d5e-4f76-a972-59d416230e95 kind: Event lastTimestamp: "2026-04-24T21:36:02Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:02Z" name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk.18a9689f8ddde812 namespace: kserve-ci-e2e-test resourceVersion: "13771" uid: bdd3e5d7-6cba-4f9a-ba93-f4d9288d411b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk namespace: kserve-ci-e2e-test resourceVersion: "13667" uid: 0a1cfccd-3d5e-4f76-a972-59d416230e95 kind: Event lastTimestamp: "2026-04-24T21:36:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:02Z" name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk.18a9689f914949ea namespace: kserve-ci-e2e-test resourceVersion: "13776" uid: f3ed84b8-e9a2-4dd7-8db0-b25a29c3f91a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk namespace: kserve-ci-e2e-test resourceVersion: "13667" uid: 0a1cfccd-3d5e-4f76-a972-59d416230e95 kind: Event lastTimestamp: "2026-04-24T21:36:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:02Z" name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk.18a9689f91acd09f namespace: kserve-ci-e2e-test resourceVersion: "13779" uid: 257fd565-a478-4878-ba87-2f7b1d313105 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk namespace: kserve-ci-e2e-test resourceVersion: "13667" uid: 0a1cfccd-3d5e-4f76-a972-59d416230e95 kind: Event lastTimestamp: "2026-04-24T21:36:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:02Z" name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk.18a9689f91b9fd5b namespace: kserve-ci-e2e-test resourceVersion: "13780" uid: c59c5813-e3b5-4ded-a9aa-a08b371a476c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk namespace: kserve-ci-e2e-test resourceVersion: "13667" uid: 0a1cfccd-3d5e-4f76-a972-59d416230e95 kind: Event lastTimestamp: "2026-04-24T21:36:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:02Z" name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk.18a9689f94aca0f1 namespace: kserve-ci-e2e-test resourceVersion: "13783" uid: 70590791-6128-49ab-a842-b34f547e51b6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk namespace: kserve-ci-e2e-test resourceVersion: "13667" uid: 0a1cfccd-3d5e-4f76-a972-59d416230e95 kind: Event lastTimestamp: "2026-04-24T21:36:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:02Z" name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk.18a9689f951173bc namespace: kserve-ci-e2e-test resourceVersion: "13784" uid: 98d4eb4c-9870-406d-870d-a8ca2712658f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:36:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk namespace: kserve-ci-e2e-test resourceVersion: "13667" uid: 0a1cfccd-3d5e-4f76-a972-59d416230e95 kind: Event lastTimestamp: "2026-04-24T21:37:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:36:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:49Z" name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk.18a9689fc9ee8962 namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: 0aa07aa1-1e69-4d52-94e9-c9ae0e700059 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk namespace: kserve-ci-e2e-test resourceVersion: "13667" uid: 0a1cfccd-3d5e-4f76-a972-59d416230e95 kind: Event lastTimestamp: "2026-04-24T21:37:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:37:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:47Z" name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk.18a968b80716b5aa namespace: kserve-ci-e2e-test resourceVersion: "14691" uid: 454393f0-6512-4ae5-92ec-4f0662456716 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk namespace: kserve-ci-e2e-test resourceVersion: "13667" uid: 0a1cfccd-3d5e-4f76-a972-59d416230e95 kind: Event lastTimestamp: "2026-04-24T21:37:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:37:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:47Z" name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk.18a968b807172d58 namespace: kserve-ci-e2e-test resourceVersion: "14694" uid: 92bd272d-1353-4f7d-a1ff-4b03fad416de reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk namespace: kserve-ci-e2e-test resourceVersion: "13667" uid: 0a1cfccd-3d5e-4f76-a972-59d416230e95 kind: Event lastTimestamp: "2026-04-24T21:37:49Z" message: 'Readiness probe failed: Get "https://10.133.0.21:8643/healthz": dial tcp 10.133.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:49Z" name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk.18a968b8783b091b namespace: kserve-ci-e2e-test resourceVersion: "14794" uid: a1c19308-bcb6-44d5-921c-80016867ee98 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb namespace: kserve-ci-e2e-test resourceVersion: "13647" uid: 904eeef0-4abd-4816-8649-f95c82b65d3d kind: Event lastTimestamp: "2026-04-24T21:35:57Z" message: 'Created pod: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb-bvczk' metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:57Z" name: isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb.18a9689e59c65700 namespace: kserve-ci-e2e-test resourceVersion: "13666" uid: b4bd7a08-9767-450d-b85c-2de17a5775a2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-7bb74-predictor namespace: kserve-ci-e2e-test resourceVersion: "13644" uid: dcf54c73-5878-482b-9e6c-ca6b143ad070 kind: Event lastTimestamp: "2026-04-24T21:35:57Z" message: Scaled up replica set isvc-sklearn-graph-raw-7bb74-predictor-76f8949cdb from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:57Z" name: isvc-sklearn-graph-raw-7bb74-predictor.18a9689e559ef149 namespace: kserve-ci-e2e-test resourceVersion: "13650" uid: 97a09424-67ad-41d4-900c-41a7066f1d55 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:36:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-7bb74-predictor namespace: kserve-ci-e2e-test resourceVersion: "13646" uid: a0f54474-3821-4004-8406-67abe1e6a0bf kind: Event lastTimestamp: "2026-04-24T21:36:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:42Z" name: isvc-sklearn-graph-raw-7bb74-predictor.18a968a1d4173561 namespace: kserve-ci-e2e-test resourceVersion: "14104" uid: bb46cfb8-c9ae-4a65-aed5-497071740002 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:36:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-7bb74-predictor namespace: kserve-ci-e2e-test resourceVersion: "13646" uid: a0f54474-3821-4004-8406-67abe1e6a0bf kind: Event lastTimestamp: "2026-04-24T21:36:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:42Z" name: isvc-sklearn-graph-raw-7bb74-predictor.18a968a1d47aabab namespace: kserve-ci-e2e-test resourceVersion: "14105" uid: 1c17f3b5-8d0b-445c-bf39-04e1add39b83 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:36:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-7bb74-predictor namespace: kserve-ci-e2e-test resourceVersion: "13866" uid: a0f54474-3821-4004-8406-67abe1e6a0bf kind: Event lastTimestamp: "2026-04-24T21:37:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:42Z" name: isvc-sklearn-graph-raw-7bb74-predictor.18a968a553750f22 namespace: kserve-ci-e2e-test resourceVersion: "14587" uid: 401d5979-e888-4bae-ad85-c18e4e55bb89 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:36:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-7bb74-predictor namespace: kserve-ci-e2e-test resourceVersion: "13866" uid: a0f54474-3821-4004-8406-67abe1e6a0bf kind: Event lastTimestamp: "2026-04-24T21:37:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:42Z" name: isvc-sklearn-graph-raw-7bb74-predictor.18a968a553dfea87 namespace: kserve-ci-e2e-test resourceVersion: "14588" uid: a70ca181-9e3f-49c0-bf9d-0072f185a266 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:35:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-7bb74 namespace: kserve-ci-e2e-test resourceVersion: "13619" uid: 10117e7f-270d-4c93-8c55-fbfed014e577 kind: Event lastTimestamp: "2026-04-24T21:37:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:37:09Z" name: isvc-sklearn-graph-raw-7bb74.18a9689e55802c03 namespace: kserve-ci-e2e-test resourceVersion: "14315" uid: da543bac-e5ef-4c29-a9e6-9b223dc4bb58 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-7bb74 namespace: kserve-ci-e2e-test resourceVersion: "13619" uid: 10117e7f-270d-4c93-8c55-fbfed014e577 kind: Event lastTimestamp: "2026-04-24T21:35:57Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-7bb74": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-7bb74": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:35:57Z" name: isvc-sklearn-graph-raw-7bb74.18a9689e55dc43d9 namespace: kserve-ci-e2e-test resourceVersion: "13653" uid: f9728289-0a98-4734-8e92-f6e99b624aaa reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-7bb74 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 10117e7f-270d-4c93-8c55-fbfed014e577 kind: Event lastTimestamp: "2026-04-24T21:37:09Z" message: InferenceService [isvc-sklearn-graph-raw-7bb74] is Ready metadata: creationTimestamp: "2026-04-24T21:37:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:37:09Z" name: isvc-sklearn-graph-raw-7bb74.18a968af2d5c6b40 namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 2c17b272-85aa-4653-958e-69a2ec4fd31c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:37:47.640222Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524 namespace: kserve-ci-e2e-test resourceVersion: "14700" uid: 6f3f4204-24d6-4d92-ba57-df58612146cf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524 to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-24T21:37:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:37:47Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524.18a968b80ae21709 namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: 389d01a5-c82c-4ab4-a9c1-cd35fb0e0c7d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6f5f9cfdf5-qjgtv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:48Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524 namespace: kserve-ci-e2e-test resourceVersion: "14726" uid: 6f3f4204-24d6-4d92-ba57-df58612146cf kind: Event lastTimestamp: "2026-04-24T21:37:48Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:37:48Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524.18a968b8245cb121 namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: 8d09d550-414a-4a9f-8251-14b6718341c2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 6f3f4204-24d6-4d92-ba57-df58612146cf kind: Event lastTimestamp: "2026-04-24T21:37:48Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b2a48068363ac4e806775467ef2dc5b6c858faecf49ae1f8bafc4262d036f97e" already present on machine metadata: creationTimestamp: "2026-04-24T21:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:48Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524.18a968b825d59fa5 namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: 397508e8-0038-4fe7-8104-f8c3ceb5e4a6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 6f3f4204-24d6-4d92-ba57-df58612146cf kind: Event lastTimestamp: "2026-04-24T21:37:48Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:48Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524.18a968b8291771a4 namespace: kserve-ci-e2e-test resourceVersion: "14767" uid: 0762ba1f-6585-434e-844d-41533531d26e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 6f3f4204-24d6-4d92-ba57-df58612146cf kind: Event lastTimestamp: "2026-04-24T21:37:48Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:48Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524.18a968b829a77404 namespace: kserve-ci-e2e-test resourceVersion: "14768" uid: 46dbe6b4-85c5-4411-a2a9-d0928cfd06e3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 6f3f4204-24d6-4d92-ba57-df58612146cf kind: Event lastTimestamp: "2026-04-24T21:37:52Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:52Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524.18a968b94070d2f3 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 6025628a-39b7-4fbc-8e3a-bd79a6005e97 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 6f3f4204-24d6-4d92-ba57-df58612146cf kind: Event lastTimestamp: "2026-04-24T21:37:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:52Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524.18a968b94546165d namespace: kserve-ci-e2e-test resourceVersion: "14835" uid: cb7439f3-e4d6-4c10-a272-e22fa25c1685 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 6f3f4204-24d6-4d92-ba57-df58612146cf kind: Event lastTimestamp: "2026-04-24T21:37:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:52Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524.18a968b945ca0a53 namespace: kserve-ci-e2e-test resourceVersion: "14837" uid: 042ea985-d768-4a4a-8021-4b4d649eb119 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 6f3f4204-24d6-4d92-ba57-df58612146cf kind: Event lastTimestamp: "2026-04-24T21:37:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:52Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524.18a968b945d88a55 namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: 25972365-b536-43c8-b704-5492de37254b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 6f3f4204-24d6-4d92-ba57-df58612146cf kind: Event lastTimestamp: "2026-04-24T21:37:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:52Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524.18a968b948cc1fe9 namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: 8ccd0741-9d7f-4a46-892f-447904fe8591 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 6f3f4204-24d6-4d92-ba57-df58612146cf kind: Event lastTimestamp: "2026-04-24T21:37:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:52Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524.18a968b9495caad6 namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: 864dcfd6-ac7f-4d3a-a3fe-8b46ac065f4b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:37:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 6f3f4204-24d6-4d92-ba57-df58612146cf kind: Event lastTimestamp: "2026-04-24T21:39:29Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:37:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:29Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524.18a968b97c9a4d67 namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: 82e49cf0-3800-4fe3-ba12-a364ed33a134 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 6f3f4204-24d6-4d92-ba57-df58612146cf kind: Event lastTimestamp: "2026-04-24T21:39:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:27Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524.18a968cf63832110 namespace: kserve-ci-e2e-test resourceVersion: "15680" uid: e8659624-b071-41bb-913c-a40f6181b146 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 6f3f4204-24d6-4d92-ba57-df58612146cf kind: Event lastTimestamp: "2026-04-24T21:39:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:27Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524.18a968cf63839f99 namespace: kserve-ci-e2e-test resourceVersion: "15681" uid: 11f9c4db-0860-4cf4-90cf-31ee5a81cfc3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 6f3f4204-24d6-4d92-ba57-df58612146cf kind: Event lastTimestamp: "2026-04-24T21:39:29Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:39:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:29Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524.18a968cfd6dc8bbb namespace: kserve-ci-e2e-test resourceVersion: "15718" uid: 601cfce8-7ee5-4817-8b04-316e95016ee6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846 namespace: kserve-ci-e2e-test resourceVersion: "14681" uid: 6e3538fe-6a59-4b80-b59b-5dd283b453a5 kind: Event lastTimestamp: "2026-04-24T21:37:47Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846-c4524' metadata: creationTimestamp: "2026-04-24T21:37:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:47Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846.18a968b80a64469d namespace: kserve-ci-e2e-test resourceVersion: "14703" uid: b465793b-1c9f-4b86-b4a1-7f66e39ca7d8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-21211-predictor namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: 55cd6c98-fd7c-4518-b03b-dc9db0d12864 kind: Event lastTimestamp: "2026-04-24T21:37:47Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-21211-predictor-5f96d8846 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:37:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:47Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor.18a968b805cad8bb namespace: kserve-ci-e2e-test resourceVersion: "14683" uid: 28d01029-1873-44c0-a076-bcd4dc6418a6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:38:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-21211-predictor namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 73035726-eddd-4cf4-97eb-4034236dcbf3 kind: Event lastTimestamp: "2026-04-24T21:38:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:32Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor.18a968bb847deb5d namespace: kserve-ci-e2e-test resourceVersion: "15151" uid: 21e5438b-312f-4ddc-9253-dce3a81899cb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:38:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-21211-predictor namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 73035726-eddd-4cf4-97eb-4034236dcbf3 kind: Event lastTimestamp: "2026-04-24T21:38:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:32Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor.18a968bb84e8e0b6 namespace: kserve-ci-e2e-test resourceVersion: "15152" uid: 00bcb981-8e87-4955-b7be-135117aebcc6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:38:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-21211-predictor namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: 73035726-eddd-4cf4-97eb-4034236dcbf3 kind: Event lastTimestamp: "2026-04-24T21:39:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:17Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor.18a968c601c45d35 namespace: kserve-ci-e2e-test resourceVersion: "15510" uid: e238492b-69e7-485a-8ff5-aa082d9ef10d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:38:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-21211-predictor namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: 73035726-eddd-4cf4-97eb-4034236dcbf3 kind: Event lastTimestamp: "2026-04-24T21:39:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:17Z" name: isvc-sklearn-graph-raw-hpa-21211-predictor.18a968c6022d6626 namespace: kserve-ci-e2e-test resourceVersion: "15515" uid: 92fa3bab-3ff4-4716-849a-a5c1f8de7a12 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:37:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-21211 namespace: kserve-ci-e2e-test resourceVersion: "14657" uid: 3e072a85-052b-49a9-bc2d-7695760d25fb kind: Event lastTimestamp: "2026-04-24T21:39:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:37:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:00Z" name: isvc-sklearn-graph-raw-hpa-21211.18a968b805d3ac6a namespace: kserve-ci-e2e-test resourceVersion: "15366" uid: 72ed3d75-12cd-4173-a196-3f509ac9efcf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-21211 namespace: kserve-ci-e2e-test resourceVersion: "14657" uid: 3e072a85-052b-49a9-bc2d-7695760d25fb kind: Event lastTimestamp: "2026-04-24T21:37:47Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-21211": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-21211": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:37:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:37:47Z" name: isvc-sklearn-graph-raw-hpa-21211.18a968b80676321c namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 00c5ba20-d2f1-4700-9208-0d300c6ef41c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-21211 namespace: kserve-ci-e2e-test resourceVersion: "15363" uid: 3e072a85-052b-49a9-bc2d-7695760d25fb kind: Event lastTimestamp: "2026-04-24T21:38:59Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-21211] is Ready metadata: creationTimestamp: "2026-04-24T21:38:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:38:59Z" name: isvc-sklearn-graph-raw-hpa-21211.18a968c8e2ea58e4 namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 8bac95af-5904-4a16-9501-6e3c4d840188 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:41:13.218816Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: e7a6b280-586b-4d38-8a28-672f8fb0c8f7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-24T21:41:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:41:13Z" name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd.18a968e7e8527b5c namespace: kserve-ci-e2e-test resourceVersion: "16633" uid: bedd8d52-467f-4815-94cf-b1e45277714a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6f5f9cfdf5-qjgtv source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:41:13Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: e7a6b280-586b-4d38-8a28-672f8fb0c8f7 kind: Event lastTimestamp: "2026-04-24T21:41:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-9e9e5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:41:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:13Z" name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd.18a968e7f0baaae4 namespace: kserve-ci-e2e-test resourceVersion: "16654" uid: 26d7e7bf-2155-4258-975c-f663e10ee635 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:15Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: e7a6b280-586b-4d38-8a28-672f8fb0c8f7 kind: Event lastTimestamp: "2026-04-24T21:41:15Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:41:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:41:15Z" name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd.18a968e85ac73e1e namespace: kserve-ci-e2e-test resourceVersion: "16666" uid: df09bf24-706b-459c-94e7-7421ffda147b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: e7a6b280-586b-4d38-8a28-672f8fb0c8f7 kind: Event lastTimestamp: "2026-04-24T21:41:15Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b2a48068363ac4e806775467ef2dc5b6c858faecf49ae1f8bafc4262d036f97e" already present on machine metadata: creationTimestamp: "2026-04-24T21:41:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:15Z" name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd.18a968e85c345f81 namespace: kserve-ci-e2e-test resourceVersion: "16669" uid: 4dbcce20-dcdd-4320-94f4-a7528a2455ec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: e7a6b280-586b-4d38-8a28-672f8fb0c8f7 kind: Event lastTimestamp: "2026-04-24T21:41:15Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:41:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:15Z" name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd.18a968e85f75f2d7 namespace: kserve-ci-e2e-test resourceVersion: "16671" uid: e86696cd-43eb-41b2-9ad5-84ce4f536c71 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: e7a6b280-586b-4d38-8a28-672f8fb0c8f7 kind: Event lastTimestamp: "2026-04-24T21:41:15Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:41:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:15Z" name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd.18a968e85fe57129 namespace: kserve-ci-e2e-test resourceVersion: "16673" uid: 7ebb4485-776b-4926-9d55-3d95a6b3810b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: e7a6b280-586b-4d38-8a28-672f8fb0c8f7 kind: Event lastTimestamp: "2026-04-24T21:41:19Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:19Z" name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd.18a968e95d3f74c7 namespace: kserve-ci-e2e-test resourceVersion: "16708" uid: 592c14cd-121b-4bd2-8644-3561ff014820 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: e7a6b280-586b-4d38-8a28-672f8fb0c8f7 kind: Event lastTimestamp: "2026-04-24T21:41:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:19Z" name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd.18a968e960217369 namespace: kserve-ci-e2e-test resourceVersion: "16710" uid: 180e22fd-18f5-4ac6-9a62-cb87cf90436f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: e7a6b280-586b-4d38-8a28-672f8fb0c8f7 kind: Event lastTimestamp: "2026-04-24T21:41:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:19Z" name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd.18a968e96091a126 namespace: kserve-ci-e2e-test resourceVersion: "16711" uid: 4b86795f-cde3-4212-b5cc-412c3777b54f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: e7a6b280-586b-4d38-8a28-672f8fb0c8f7 kind: Event lastTimestamp: "2026-04-24T21:41:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:19Z" name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd.18a968e9609d81eb namespace: kserve-ci-e2e-test resourceVersion: "16714" uid: 66381d7e-baa6-485a-89bb-ed57a5e5ca13 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: e7a6b280-586b-4d38-8a28-672f8fb0c8f7 kind: Event lastTimestamp: "2026-04-24T21:41:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:19Z" name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd.18a968e96389c07b namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: 2d1c2ea0-563f-480a-bbb7-aea2c54cd81c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: e7a6b280-586b-4d38-8a28-672f8fb0c8f7 kind: Event lastTimestamp: "2026-04-24T21:41:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:19Z" name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd.18a968e9640d9d22 namespace: kserve-ci-e2e-test resourceVersion: "16717" uid: 18c473cd-8ed2-4540-8b67-3e091a785d42 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 17 eventTime: null firstTimestamp: "2026-04-24T21:41:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: e7a6b280-586b-4d38-8a28-672f8fb0c8f7 kind: Event lastTimestamp: "2026-04-24T21:43:51Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:41:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:51Z" name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd.18a968e99934ecec namespace: kserve-ci-e2e-test resourceVersion: "17889" uid: bbe8c235-69b4-41ff-aaa7-f1409bacf905 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: e7a6b280-586b-4d38-8a28-672f8fb0c8f7 kind: Event lastTimestamp: "2026-04-24T21:43:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:43Z" name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd.18a9690adf90c16d namespace: kserve-ci-e2e-test resourceVersion: "17771" uid: 09ea76f6-f094-449a-98e1-d12473eb481d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: e7a6b280-586b-4d38-8a28-672f8fb0c8f7 kind: Event lastTimestamp: "2026-04-24T21:43:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:43Z" name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd.18a9690adf914c94 namespace: kserve-ci-e2e-test resourceVersion: "17773" uid: 81af4ef0-d5bc-4b76-af8a-206bb18c6f56 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:43:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: e7a6b280-586b-4d38-8a28-672f8fb0c8f7 kind: Event lastTimestamp: "2026-04-24T21:43:51Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:43:51Z" name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd.18a9690b97b257fe namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 2603fdf5-02dc-48fc-bbe8-a9ea2f21fa31 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd namespace: kserve-ci-e2e-test resourceVersion: "16600" uid: 21836997-123c-432a-94a0-2fb250de6e15 kind: Event lastTimestamp: "2026-04-24T21:41:13Z" message: 'Created pod: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd-nqbvd' metadata: creationTimestamp: "2026-04-24T21:41:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:13Z" name: isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd.18a968e7e7e5396c namespace: kserve-ci-e2e-test resourceVersion: "16627" uid: 069482cc-c178-44d2-87dc-5a8abf25f9e6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-9e9e5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16597" uid: c652e43b-3fdc-47b8-8e3d-7eed71e590f4 kind: Event lastTimestamp: "2026-04-24T21:41:13Z" message: Scaled up replica set isvc-sklearn-scale-raw-9e9e5-predictor-db45f49bd from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:41:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:13Z" name: isvc-sklearn-scale-raw-9e9e5-predictor.18a968e7e486ac0d namespace: kserve-ci-e2e-test resourceVersion: "16607" uid: 13056ddd-d00c-419f-a9a8-808623be6c69 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-9e9e5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16599" uid: bf77432c-c26e-4573-84fe-3e2962d482bd kind: Event lastTimestamp: "2026-04-24T21:41:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:41:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:28Z" name: isvc-sklearn-scale-raw-9e9e5-predictor.18a968eb6355dc1b namespace: kserve-ci-e2e-test resourceVersion: "16788" uid: 0c561e83-9259-4275-96a4-272a1a6ba455 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-9e9e5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16599" uid: bf77432c-c26e-4573-84fe-3e2962d482bd kind: Event lastTimestamp: "2026-04-24T21:41:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:41:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:28Z" name: isvc-sklearn-scale-raw-9e9e5-predictor.18a968eb63c2b55b namespace: kserve-ci-e2e-test resourceVersion: "16790" uid: 1159b8f0-062c-4438-a0db-3d933a0bee8f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:41:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-9e9e5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16789" uid: bf77432c-c26e-4573-84fe-3e2962d482bd kind: Event lastTimestamp: "2026-04-24T21:43:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:43:43Z" name: isvc-sklearn-scale-raw-9e9e5-predictor.18a968eee25ec6b9 namespace: kserve-ci-e2e-test resourceVersion: "17742" uid: 91428ab6-abc9-453a-b3de-3900e1dc8506 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:41:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-9e9e5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16789" uid: bf77432c-c26e-4573-84fe-3e2962d482bd kind: Event lastTimestamp: "2026-04-24T21:43:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:43:43Z" name: isvc-sklearn-scale-raw-9e9e5-predictor.18a968eee2ca932e namespace: kserve-ci-e2e-test resourceVersion: "17744" uid: db2b4e66-3bed-4654-8034-8823667531cf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-24T21:41:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-9e9e5 namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: b15c2e6f-7b1d-41ae-b545-1a97d9a9217c kind: Event lastTimestamp: "2026-04-24T21:43:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:41:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:43:41Z" name: isvc-sklearn-scale-raw-9e9e5.18a968e7e458e2cd namespace: kserve-ci-e2e-test resourceVersion: "17728" uid: 1c5cb6eb-5a7d-44c5-abfb-78cd722fe157 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-9e9e5 namespace: kserve-ci-e2e-test resourceVersion: "16606" uid: b15c2e6f-7b1d-41ae-b545-1a97d9a9217c kind: Event lastTimestamp: "2026-04-24T21:41:13Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-9e9e5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-9e9e5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:41:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:41:13Z" name: isvc-sklearn-scale-raw-9e9e5.18a968e7e8910484 namespace: kserve-ci-e2e-test resourceVersion: "16635" uid: c92a79d7-6303-423a-a3e2-594f2f3024cf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:43:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-9e9e5 namespace: kserve-ci-e2e-test resourceVersion: "17726" uid: b15c2e6f-7b1d-41ae-b545-1a97d9a9217c kind: Event lastTimestamp: "2026-04-24T21:43:41Z" message: InferenceService [isvc-sklearn-scale-raw-9e9e5] is Ready metadata: creationTimestamp: "2026-04-24T21:43:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:43:41Z" name: isvc-sklearn-scale-raw-9e9e5.18a9690a83f35840 namespace: kserve-ci-e2e-test resourceVersion: "17727" uid: be60cbc9-55c7-4cbe-8c3b-7a5ef8ee46cf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:35:57.393399Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t namespace: kserve-ci-e2e-test resourceVersion: "13693" uid: 90218cfe-6610-442a-9cdf-aedb36d45ffe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:35:57Z" name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t.18a9689e5fa90b03 namespace: kserve-ci-e2e-test resourceVersion: "13697" uid: 2d138af3-275d-4eb6-9ecc-68345794eabd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6f5f9cfdf5-qjgtv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t namespace: kserve-ci-e2e-test resourceVersion: "13695" uid: 90218cfe-6610-442a-9cdf-aedb36d45ffe kind: Event lastTimestamp: "2026-04-24T21:35:57Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-7bb74-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:57Z" name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t.18a9689e6907648c namespace: kserve-ci-e2e-test resourceVersion: "13705" uid: 08d6332c-6349-42ae-971f-a742dfa8e98b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t namespace: kserve-ci-e2e-test resourceVersion: "13704" uid: 90218cfe-6610-442a-9cdf-aedb36d45ffe kind: Event lastTimestamp: "2026-04-24T21:35:58Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:35:58Z" name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t.18a9689e9bddee5c namespace: kserve-ci-e2e-test resourceVersion: "13721" uid: bbd3d166-cb97-4f8f-9d74-3b4ea5172f5b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t namespace: kserve-ci-e2e-test resourceVersion: "13695" uid: 90218cfe-6610-442a-9cdf-aedb36d45ffe kind: Event lastTimestamp: "2026-04-24T21:35:58Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b2a48068363ac4e806775467ef2dc5b6c858faecf49ae1f8bafc4262d036f97e" already present on machine metadata: creationTimestamp: "2026-04-24T21:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:58Z" name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t.18a9689e9d454214 namespace: kserve-ci-e2e-test resourceVersion: "13723" uid: e1836f12-e772-4287-a02e-1e6df05ab390 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t namespace: kserve-ci-e2e-test resourceVersion: "13695" uid: 90218cfe-6610-442a-9cdf-aedb36d45ffe kind: Event lastTimestamp: "2026-04-24T21:35:58Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:58Z" name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t.18a9689e9feec5d1 namespace: kserve-ci-e2e-test resourceVersion: "13727" uid: 48b52f7e-980d-471f-8f15-9ca7b05f3938 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t namespace: kserve-ci-e2e-test resourceVersion: "13695" uid: 90218cfe-6610-442a-9cdf-aedb36d45ffe kind: Event lastTimestamp: "2026-04-24T21:35:58Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:58Z" name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t.18a9689ea04a23c9 namespace: kserve-ci-e2e-test resourceVersion: "13729" uid: 4c0ecc92-452d-47c0-818b-2c236d5cb263 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t namespace: kserve-ci-e2e-test resourceVersion: "13695" uid: 90218cfe-6610-442a-9cdf-aedb36d45ffe kind: Event lastTimestamp: "2026-04-24T21:36:02Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-24T21:36:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:02Z" name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t.18a9689f8dfc1745 namespace: kserve-ci-e2e-test resourceVersion: "13772" uid: 1a65f35e-7e88-4b2c-b827-c782779b1f01 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t namespace: kserve-ci-e2e-test resourceVersion: "13695" uid: 90218cfe-6610-442a-9cdf-aedb36d45ffe kind: Event lastTimestamp: "2026-04-24T21:36:19Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 17.166s (17.166s including waiting). Image size: 1306417402 bytes.' metadata: creationTimestamp: "2026-04-24T21:36:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:19Z" name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t.18a968a38d29cf19 namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: 97cf5c45-d645-4ff5-af85-50ea70ee0227 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t namespace: kserve-ci-e2e-test resourceVersion: "13695" uid: 90218cfe-6610-442a-9cdf-aedb36d45ffe kind: Event lastTimestamp: "2026-04-24T21:36:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:36:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:19Z" name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t.18a968a38ffb0b92 namespace: kserve-ci-e2e-test resourceVersion: "13927" uid: 22bd43b4-9eb4-43e8-962a-b70c96fb3b0b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t namespace: kserve-ci-e2e-test resourceVersion: "13695" uid: 90218cfe-6610-442a-9cdf-aedb36d45ffe kind: Event lastTimestamp: "2026-04-24T21:36:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:36:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:19Z" name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t.18a968a390578008 namespace: kserve-ci-e2e-test resourceVersion: "13928" uid: cbdb3aed-ab21-46eb-a8c5-13ae96f75c1d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t namespace: kserve-ci-e2e-test resourceVersion: "13695" uid: 90218cfe-6610-442a-9cdf-aedb36d45ffe kind: Event lastTimestamp: "2026-04-24T21:36:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:36:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:19Z" name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t.18a968a39061b3eb namespace: kserve-ci-e2e-test resourceVersion: "13929" uid: eaf0260b-c3f9-49ef-ba53-ad2dc057926e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t namespace: kserve-ci-e2e-test resourceVersion: "13695" uid: 90218cfe-6610-442a-9cdf-aedb36d45ffe kind: Event lastTimestamp: "2026-04-24T21:36:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:36:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:19Z" name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t.18a968a392dc9c46 namespace: kserve-ci-e2e-test resourceVersion: "13930" uid: 07b50add-71c7-4fed-8b03-9b94caa2aad0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t namespace: kserve-ci-e2e-test resourceVersion: "13695" uid: 90218cfe-6610-442a-9cdf-aedb36d45ffe kind: Event lastTimestamp: "2026-04-24T21:36:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:36:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:19Z" name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t.18a968a393381a05 namespace: kserve-ci-e2e-test resourceVersion: "13931" uid: 9db4a028-ac9d-429c-b392-4892518f2e30 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T21:36:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t namespace: kserve-ci-e2e-test resourceVersion: "13695" uid: 90218cfe-6610-442a-9cdf-aedb36d45ffe kind: Event lastTimestamp: "2026-04-24T21:37:17Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:36:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:17Z" name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t.18a968a3fe6ec73a namespace: kserve-ci-e2e-test resourceVersion: "14372" uid: 7b72f423-4c09-44a9-951c-d4c5be7b89b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t namespace: kserve-ci-e2e-test resourceVersion: "13695" uid: 90218cfe-6610-442a-9cdf-aedb36d45ffe kind: Event lastTimestamp: "2026-04-24T21:37:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:37:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:47Z" name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t.18a968b80e4f964f namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 985d20c0-76b2-42a4-afc9-d3fa4706b44b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t namespace: kserve-ci-e2e-test resourceVersion: "13695" uid: 90218cfe-6610-442a-9cdf-aedb36d45ffe kind: Event lastTimestamp: "2026-04-24T21:37:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:37:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:47Z" name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t.18a968b80e500ce2 namespace: kserve-ci-e2e-test resourceVersion: "14743" uid: 730e6bae-70dd-4f1c-b3d7-4cdab7623798 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585 namespace: kserve-ci-e2e-test resourceVersion: "13681" uid: 2519022b-9def-4802-bc43-af490d937cbd kind: Event lastTimestamp: "2026-04-24T21:35:57Z" message: 'Created pod: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585-w9q5t' metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:57Z" name: isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585.18a9689e5f438d9e namespace: kserve-ci-e2e-test resourceVersion: "13694" uid: 86ece31f-dc48-486d-906a-54dd7ef4b75c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-7bb74-predictor namespace: kserve-ci-e2e-test resourceVersion: "13678" uid: 0a5019bc-503f-4a30-97e2-176f2f9de59f kind: Event lastTimestamp: "2026-04-24T21:35:57Z" message: Scaled up replica set isvc-xgboost-graph-raw-7bb74-predictor-7bf6b85585 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:57Z" name: isvc-xgboost-graph-raw-7bb74-predictor.18a9689e5c769d01 namespace: kserve-ci-e2e-test resourceVersion: "13685" uid: ac865d92-3a8f-4957-a302-324dcec280d8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:36:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-7bb74-predictor namespace: kserve-ci-e2e-test resourceVersion: "13682" uid: 1fba83fd-a03d-477e-b449-1aaf5a592a49 kind: Event lastTimestamp: "2026-04-24T21:36:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:57Z" name: isvc-xgboost-graph-raw-7bb74-predictor.18a968a1db0e702e namespace: kserve-ci-e2e-test resourceVersion: "14217" uid: 24881564-f723-4cdf-80f1-ed96a80d2a64 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:36:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-7bb74-predictor namespace: kserve-ci-e2e-test resourceVersion: "13682" uid: 1fba83fd-a03d-477e-b449-1aaf5a592a49 kind: Event lastTimestamp: "2026-04-24T21:36:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:57Z" name: isvc-xgboost-graph-raw-7bb74-predictor.18a968a1dbbdae6b namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: 7e11e08b-84a1-492e-97a5-71639f927c15 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:37:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-7bb74-predictor namespace: kserve-ci-e2e-test resourceVersion: "13873" uid: 1fba83fd-a03d-477e-b449-1aaf5a592a49 kind: Event lastTimestamp: "2026-04-24T21:37:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:37:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:42Z" name: isvc-xgboost-graph-raw-7bb74-predictor.18a968afd6a6aed2 namespace: kserve-ci-e2e-test resourceVersion: "14589" uid: 380d36b6-2bfc-4cc5-8b35-18c505814346 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:37:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-7bb74-predictor namespace: kserve-ci-e2e-test resourceVersion: "13873" uid: 1fba83fd-a03d-477e-b449-1aaf5a592a49 kind: Event lastTimestamp: "2026-04-24T21:37:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:37:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:42Z" name: isvc-xgboost-graph-raw-7bb74-predictor.18a968afd71c910f namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: eb96e752-cfdc-44fc-9392-2c636219c2e1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T21:35:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-7bb74 namespace: kserve-ci-e2e-test resourceVersion: "13659" uid: e3040fa3-2102-4eeb-97d4-6f8bd491e452 kind: Event lastTimestamp: "2026-04-24T21:37:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:37:27Z" name: isvc-xgboost-graph-raw-7bb74.18a9689e5c7ecad6 namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: 6c6b79aa-01f0-4bb7-8801-20c2806c21af reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-7bb74 namespace: kserve-ci-e2e-test resourceVersion: "13659" uid: e3040fa3-2102-4eeb-97d4-6f8bd491e452 kind: Event lastTimestamp: "2026-04-24T21:35:57Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-7bb74": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-7bb74": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:35:57Z" name: isvc-xgboost-graph-raw-7bb74.18a9689e5d480fcb namespace: kserve-ci-e2e-test resourceVersion: "13689" uid: fe3534b0-ca4b-408b-9873-c23bf17502c2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-7bb74 namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: e3040fa3-2102-4eeb-97d4-6f8bd491e452 kind: Event lastTimestamp: "2026-04-24T21:37:27Z" message: InferenceService [isvc-xgboost-graph-raw-7bb74] is Ready metadata: creationTimestamp: "2026-04-24T21:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:37:27Z" name: isvc-xgboost-graph-raw-7bb74.18a968b362e890b9 namespace: kserve-ci-e2e-test resourceVersion: "14454" uid: 01d22156-c3c1-4122-abe4-af1fafaa82c2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:37:47.734264Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj namespace: kserve-ci-e2e-test resourceVersion: "14746" uid: fd3e79e1-ba1d-4249-a3fc-c46bfd68dce5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-24T21:37:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:37:47Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj.18a968b8107d27aa namespace: kserve-ci-e2e-test resourceVersion: "14752" uid: 7504f5e5-f309-47dc-8f18-4d9dc85cf26e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6f5f9cfdf5-qjgtv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:48Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj namespace: kserve-ci-e2e-test resourceVersion: "14758" uid: fd3e79e1-ba1d-4249-a3fc-c46bfd68dce5 kind: Event lastTimestamp: "2026-04-24T21:37:48Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:37:48Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj.18a968b82a761396 namespace: kserve-ci-e2e-test resourceVersion: "14770" uid: adce8157-a9ab-4848-a7b7-9ed4483ff2a4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: fd3e79e1-ba1d-4249-a3fc-c46bfd68dce5 kind: Event lastTimestamp: "2026-04-24T21:37:48Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b2a48068363ac4e806775467ef2dc5b6c858faecf49ae1f8bafc4262d036f97e" already present on machine metadata: creationTimestamp: "2026-04-24T21:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:48Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj.18a968b82cc94696 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 1fcba581-4476-4a37-a168-bfa48cb1b211 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: fd3e79e1-ba1d-4249-a3fc-c46bfd68dce5 kind: Event lastTimestamp: "2026-04-24T21:37:48Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:48Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj.18a968b830652b06 namespace: kserve-ci-e2e-test resourceVersion: "14773" uid: 09222253-1df7-48d7-9703-d3fbed4e7795 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: fd3e79e1-ba1d-4249-a3fc-c46bfd68dce5 kind: Event lastTimestamp: "2026-04-24T21:37:48Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:48Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj.18a968b830d03d6b namespace: kserve-ci-e2e-test resourceVersion: "14774" uid: d3493c74-adea-48c6-ac03-5dbef4a2898b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: fd3e79e1-ba1d-4249-a3fc-c46bfd68dce5 kind: Event lastTimestamp: "2026-04-24T21:37:52Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-24T21:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:52Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj.18a968b9405fdf79 namespace: kserve-ci-e2e-test resourceVersion: "14825" uid: 1883b298-357f-40de-98b2-ee68b4c17588 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: fd3e79e1-ba1d-4249-a3fc-c46bfd68dce5 kind: Event lastTimestamp: "2026-04-24T21:37:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:52Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj.18a968b9454e2279 namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: 5139c4da-cf99-4455-a20f-40ea302281ba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: fd3e79e1-ba1d-4249-a3fc-c46bfd68dce5 kind: Event lastTimestamp: "2026-04-24T21:37:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:52Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj.18a968b945d9ce13 namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: 70fbdc24-d7d1-4d4e-b97f-eaa3a0bb61e8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: fd3e79e1-ba1d-4249-a3fc-c46bfd68dce5 kind: Event lastTimestamp: "2026-04-24T21:37:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:52Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj.18a968b945e78f2e namespace: kserve-ci-e2e-test resourceVersion: "14840" uid: 5bab6288-a829-434f-9673-50fec96a520d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: fd3e79e1-ba1d-4249-a3fc-c46bfd68dce5 kind: Event lastTimestamp: "2026-04-24T21:37:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:52Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj.18a968b948d66057 namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: ffd8eaa6-650f-4ecd-93d7-8102dc5403ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: fd3e79e1-ba1d-4249-a3fc-c46bfd68dce5 kind: Event lastTimestamp: "2026-04-24T21:37:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:52Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj.18a968b9495cabbf namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: b5634039-8a5c-4bac-89f3-d5bc77289fd7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:37:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: fd3e79e1-ba1d-4249-a3fc-c46bfd68dce5 kind: Event lastTimestamp: "2026-04-24T21:39:29Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:37:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:29Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj.18a968b97c7fd004 namespace: kserve-ci-e2e-test resourceVersion: "15720" uid: 159ce67c-e695-4ab9-aeb8-520cce055e92 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: fd3e79e1-ba1d-4249-a3fc-c46bfd68dce5 kind: Event lastTimestamp: "2026-04-24T21:39:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:27Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj.18a968cf601c96d0 namespace: kserve-ci-e2e-test resourceVersion: "15656" uid: 8ecc30ba-bda7-4347-b3da-80315b81dddf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: fd3e79e1-ba1d-4249-a3fc-c46bfd68dce5 kind: Event lastTimestamp: "2026-04-24T21:39:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:27Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj.18a968cf601f034b namespace: kserve-ci-e2e-test resourceVersion: "15664" uid: 7b45571e-76a8-4a23-903c-292774ef0ea5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: fd3e79e1-ba1d-4249-a3fc-c46bfd68dce5 kind: Event lastTimestamp: "2026-04-24T21:39:29Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:39:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:29Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj.18a968cfd6dba5f1 namespace: kserve-ci-e2e-test resourceVersion: "15717" uid: 9ebeac7a-56fb-42ad-a16e-aebf88b68e2d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f namespace: kserve-ci-e2e-test resourceVersion: "14713" uid: 9384915d-bd47-43c0-b944-791e42615e0c kind: Event lastTimestamp: "2026-04-24T21:37:47Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f-zr7qj' metadata: creationTimestamp: "2026-04-24T21:37:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:47Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f.18a968b80fec23b8 namespace: kserve-ci-e2e-test resourceVersion: "14748" uid: 37659972-bafc-4c0a-b696-41dbafabbca9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-21211-predictor namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: 148e40e7-d8cc-418e-bc1d-f4c68b5ff079 kind: Event lastTimestamp: "2026-04-24T21:37:47Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-21211-predictor-69cf7fd86f from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:37:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:47Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor.18a968b80b9bd31f namespace: kserve-ci-e2e-test resourceVersion: "14716" uid: 8878d16d-b60b-46eb-9715-cea448d2e815 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:38:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-21211-predictor namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: f6447515-409e-4ad0-b66f-94f8c3493899 kind: Event lastTimestamp: "2026-04-24T21:38:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:32Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor.18a968bb8a3d5ea0 namespace: kserve-ci-e2e-test resourceVersion: "15153" uid: 37f52937-557d-45a3-b658-cb418366370d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:38:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-21211-predictor namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: f6447515-409e-4ad0-b66f-94f8c3493899 kind: Event lastTimestamp: "2026-04-24T21:38:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:32Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor.18a968bb8aa61137 namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: 3443c481-0e7d-48a5-8259-8b69a78e0e70 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:38:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-21211-predictor namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: f6447515-409e-4ad0-b66f-94f8c3493899 kind: Event lastTimestamp: "2026-04-24T21:39:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:17Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor.18a968c607c4572f namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: 9cf531d8-12e8-4a82-bac3-08e217ce40ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:38:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-21211-predictor namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: f6447515-409e-4ad0-b66f-94f8c3493899 kind: Event lastTimestamp: "2026-04-24T21:39:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:17Z" name: isvc-xgboost-graph-raw-hpa-21211-predictor.18a968c60832c7b2 namespace: kserve-ci-e2e-test resourceVersion: "15525" uid: 4b57512f-e8ef-404c-bada-1b9f7aaabe1f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:37:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-21211 namespace: kserve-ci-e2e-test resourceVersion: "14695" uid: 17fe32f8-6152-435f-893d-d49066f71fec kind: Event lastTimestamp: "2026-04-24T21:39:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:37:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:00Z" name: isvc-xgboost-graph-raw-hpa-21211.18a968b80b9e3fdb namespace: kserve-ci-e2e-test resourceVersion: "15365" uid: 408f7d7d-0c01-4e10-84f0-4d44ec74a74d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-21211 namespace: kserve-ci-e2e-test resourceVersion: "15359" uid: 17fe32f8-6152-435f-893d-d49066f71fec kind: Event lastTimestamp: "2026-04-24T21:38:59Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-21211] is Ready metadata: creationTimestamp: "2026-04-24T21:38:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:38:59Z" name: isvc-xgboost-graph-raw-hpa-21211.18a968c8e061abe3 namespace: kserve-ci-e2e-test resourceVersion: "15360" uid: c5bb9690-34a8-4641-a945-85bf4d3b4701 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:39:27.934864Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28 namespace: kserve-ci-e2e-test resourceVersion: "15682" uid: 5131ad7e-9174-49ef-9b25-764060e0f422 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28 to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-24T21:39:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:39:27Z" name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28.18a968cf64e8dd0e namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: d4b1c13d-ccc3-4883-92ce-2001361078b8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6f5f9cfdf5-qjgtv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:28Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28 namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: 5131ad7e-9174-49ef-9b25-764060e0f422 kind: Event lastTimestamp: "2026-04-24T21:39:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-raw-9a20e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:39:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:28Z" name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28.18a968cf6e3ea882 namespace: kserve-ci-e2e-test resourceVersion: "15698" uid: 8c2da5ea-868a-44d6-87f5-0199f18e05a2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:28Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28 namespace: kserve-ci-e2e-test resourceVersion: "15693" uid: 5131ad7e-9174-49ef-9b25-764060e0f422 kind: Event lastTimestamp: "2026-04-24T21:39:28Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:39:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:39:28Z" name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28.18a968cfa1cac22a namespace: kserve-ci-e2e-test resourceVersion: "15707" uid: ea47f0b6-042d-4f37-91d1-73d3f4faa6a2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28 namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: 5131ad7e-9174-49ef-9b25-764060e0f422 kind: Event lastTimestamp: "2026-04-24T21:39:28Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-24T21:39:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:28Z" name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28.18a968cfa318326e namespace: kserve-ci-e2e-test resourceVersion: "15709" uid: fc0d68c8-7092-4e49-b4af-fcf4590bdebc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28 namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: 5131ad7e-9174-49ef-9b25-764060e0f422 kind: Event lastTimestamp: "2026-04-24T21:39:30Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.07s (1.07s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-24T21:39:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:30Z" name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28.18a968cfe2e5501b namespace: kserve-ci-e2e-test resourceVersion: "15723" uid: 11bc0a9d-4c8b-43bb-bf16-2d688fbe33ff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28 namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: 5131ad7e-9174-49ef-9b25-764060e0f422 kind: Event lastTimestamp: "2026-04-24T21:39:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:39:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:30Z" name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28.18a968cfe5d2b2ee namespace: kserve-ci-e2e-test resourceVersion: "15725" uid: 0d6fbd8e-1482-4965-a736-79b928f8786c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28 namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: 5131ad7e-9174-49ef-9b25-764060e0f422 kind: Event lastTimestamp: "2026-04-24T21:39:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:39:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:30Z" name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28.18a968cfe643e0ec namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: 9ff0301a-0eb7-4df3-884c-0840df8dce70 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28 namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: 5131ad7e-9174-49ef-9b25-764060e0f422 kind: Event lastTimestamp: "2026-04-24T21:39:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:39:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:30Z" name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28.18a968cfe64f5e64 namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: e80b13be-b321-4458-ac6d-90a0ef01cb93 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28 namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: 5131ad7e-9174-49ef-9b25-764060e0f422 kind: Event lastTimestamp: "2026-04-24T21:39:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:39:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:30Z" name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28.18a968cfe9735985 namespace: kserve-ci-e2e-test resourceVersion: "15728" uid: fc17dd3f-bef6-4ce7-9800-c37190e7cfc9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28 namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: 5131ad7e-9174-49ef-9b25-764060e0f422 kind: Event lastTimestamp: "2026-04-24T21:39:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:39:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:30Z" name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28.18a968cfe9e177b4 namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: 618b58be-3cbf-402e-97e3-10b8b74d8cba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28 namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: 5131ad7e-9174-49ef-9b25-764060e0f422 kind: Event lastTimestamp: "2026-04-24T21:41:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:41:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:13Z" name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28.18a968e7e3008657 namespace: kserve-ci-e2e-test resourceVersion: "16593" uid: 08907387-aef2-4efb-8b7d-a8243e39ab1a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28 namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: 5131ad7e-9174-49ef-9b25-764060e0f422 kind: Event lastTimestamp: "2026-04-24T21:41:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:41:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:13Z" name: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28.18a968e7e3010ac9 namespace: kserve-ci-e2e-test resourceVersion: "16596" uid: 85a7579d-2022-4bc7-9c99-9ac2eb1a247b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-9a20e-predictor-7bfbff9d59 namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: 0003937a-e5a2-42ba-b349-ada44a7f82c3 kind: Event lastTimestamp: "2026-04-24T21:39:27Z" message: 'Created pod: message-dumper-raw-9a20e-predictor-7bfbff9d59-cjz28' metadata: creationTimestamp: "2026-04-24T21:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:27Z" name: message-dumper-raw-9a20e-predictor-7bfbff9d59.18a968cf647d95af namespace: kserve-ci-e2e-test resourceVersion: "15683" uid: 51e44a26-0409-4825-9dbd-1167d05b6451 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-9a20e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15651" uid: f9bc6701-5223-4041-befd-50fe5b28610f kind: Event lastTimestamp: "2026-04-24T21:39:27Z" message: Scaled up replica set message-dumper-raw-9a20e-predictor-7bfbff9d59 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:27Z" name: message-dumper-raw-9a20e-predictor.18a968cf610fbea0 namespace: kserve-ci-e2e-test resourceVersion: "15662" uid: dc2c82c7-fe8a-4f93-94ec-f5de8974f4d2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:39:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-9a20e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: edf9084d-fe92-415a-9191-13292d209c9a kind: Event lastTimestamp: "2026-04-24T21:40:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:39:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:12Z" name: message-dumper-raw-9a20e-predictor.18a968d2e01bab03 namespace: kserve-ci-e2e-test resourceVersion: "16134" uid: 50483193-e879-4d39-bd54-3689339b310d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:39:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-9a20e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: edf9084d-fe92-415a-9191-13292d209c9a kind: Event lastTimestamp: "2026-04-24T21:40:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:39:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:12Z" name: message-dumper-raw-9a20e-predictor.18a968d2e08ac791 namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: 5a6ab1fc-0e6e-4b11-b150-b80edac481aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-9a20e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: edf9084d-fe92-415a-9191-13292d209c9a kind: Event lastTimestamp: "2026-04-24T21:39:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:39:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:57Z" name: message-dumper-raw-9a20e-predictor.18a968d65f8cd805 namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: f696c543-1c3a-4d11-a9a1-10f026cc6c5f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-9a20e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: edf9084d-fe92-415a-9191-13292d209c9a kind: Event lastTimestamp: "2026-04-24T21:39:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:39:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:57Z" name: message-dumper-raw-9a20e-predictor.18a968d660007ce7 namespace: kserve-ci-e2e-test resourceVersion: "16018" uid: 3d7eb5d0-0f9a-4874-92a6-cfd6f2d4381c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T21:39:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-9a20e namespace: kserve-ci-e2e-test resourceVersion: "15627" uid: fff98613-2971-46cb-919d-f21573c46998 kind: Event lastTimestamp: "2026-04-24T21:39:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:39Z" name: message-dumper-raw-9a20e.18a968cf616a8f55 namespace: kserve-ci-e2e-test resourceVersion: "15812" uid: 02dc54b2-3e5e-4b76-9d00-fda3454dc144 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-9a20e namespace: kserve-ci-e2e-test resourceVersion: "15627" uid: fff98613-2971-46cb-919d-f21573c46998 kind: Event lastTimestamp: "2026-04-24T21:39:27Z" message: 'Failed to update status for InferenceService "message-dumper-raw-9a20e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-9a20e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:27Z" name: message-dumper-raw-9a20e.18a968cf62346bf0 namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: d3d15bc3-21b7-46d6-b7e3-6cd274dc22f8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-9a20e namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: fff98613-2971-46cb-919d-f21573c46998 kind: Event lastTimestamp: "2026-04-24T21:39:39Z" message: InferenceService [message-dumper-raw-9a20e] is Ready metadata: creationTimestamp: "2026-04-24T21:39:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:39Z" name: message-dumper-raw-9a20e.18a968d2076412c6 namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: 57fa8eda-935e-4c3f-834d-e8bba77ebbc2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:37:37.387976Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-7bb74-5899c94ff8-n7d2h namespace: kserve-ci-e2e-test resourceVersion: "14530" uid: f49ae6ba-9b34-4cec-a9b7-537a3730a194 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-7bb74-5899c94ff8-n7d2h to ip-10-0-131-55.ec2.internal metadata: creationTimestamp: "2026-04-24T21:37:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:37:37Z" name: model-chainer-raw-7bb74-5899c94ff8-n7d2h.18a968b5a7cd3c8e namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: b30714e3-9b44-46cf-a8f2-883940acc09b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6f5f9cfdf5-qjgtv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:37Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-7bb74-5899c94ff8-n7d2h namespace: kserve-ci-e2e-test resourceVersion: "14538" uid: f49ae6ba-9b34-4cec-a9b7-537a3730a194 kind: Event lastTimestamp: "2026-04-24T21:37:37Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:37:37Z" name: model-chainer-raw-7bb74-5899c94ff8-n7d2h.18a968b5c0c93011 namespace: kserve-ci-e2e-test resourceVersion: "14550" uid: 18192943-d3ab-48e9-a89b-c1f31d0336ba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-7bb74} kind: Pod name: model-chainer-raw-7bb74-5899c94ff8-n7d2h namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: f49ae6ba-9b34-4cec-a9b7-537a3730a194 kind: Event lastTimestamp: "2026-04-24T21:37:37Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:b42fc184cd9997665c45700486ec1c9301c424ede94aa53d528583b0fe44a7ad" metadata: creationTimestamp: "2026-04-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:37Z" name: model-chainer-raw-7bb74-5899c94ff8-n7d2h.18a968b5c21d95c2 namespace: kserve-ci-e2e-test resourceVersion: "14552" uid: a6127a96-7390-4d73-adae-d1805988d7c2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-55.ec2.internal source: component: kubelet host: ip-10-0-131-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-7bb74} kind: Pod name: model-chainer-raw-7bb74-5899c94ff8-n7d2h namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: f49ae6ba-9b34-4cec-a9b7-537a3730a194 kind: Event lastTimestamp: "2026-04-24T21:37:39Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:b42fc184cd9997665c45700486ec1c9301c424ede94aa53d528583b0fe44a7ad" in 2.115s (2.115s including waiting). Image size: 216346205 bytes.' metadata: creationTimestamp: "2026-04-24T21:37:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:39Z" name: model-chainer-raw-7bb74-5899c94ff8-n7d2h.18a968b6403a3cf8 namespace: kserve-ci-e2e-test resourceVersion: "14567" uid: 7930e175-c02e-4e49-be9f-02dcaedd7afb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-55.ec2.internal source: component: kubelet host: ip-10-0-131-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-7bb74} kind: Pod name: model-chainer-raw-7bb74-5899c94ff8-n7d2h namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: f49ae6ba-9b34-4cec-a9b7-537a3730a194 kind: Event lastTimestamp: "2026-04-24T21:37:39Z" message: 'Created container: model-chainer-raw-7bb74' metadata: creationTimestamp: "2026-04-24T21:37:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:39Z" name: model-chainer-raw-7bb74-5899c94ff8-n7d2h.18a968b6433e35f3 namespace: kserve-ci-e2e-test resourceVersion: "14569" uid: 529afdce-a0aa-405c-a0f8-b4c8a9d48e9a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-55.ec2.internal source: component: kubelet host: ip-10-0-131-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-7bb74} kind: Pod name: model-chainer-raw-7bb74-5899c94ff8-n7d2h namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: f49ae6ba-9b34-4cec-a9b7-537a3730a194 kind: Event lastTimestamp: "2026-04-24T21:37:40Z" message: Started container model-chainer-raw-7bb74 metadata: creationTimestamp: "2026-04-24T21:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:40Z" name: model-chainer-raw-7bb74-5899c94ff8-n7d2h.18a968b643aacf47 namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: 3dc52cdb-e423-40ae-ba57-b3dfb2bb8f75 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-55.ec2.internal source: component: kubelet host: ip-10-0-131-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-7bb74} kind: Pod name: model-chainer-raw-7bb74-5899c94ff8-n7d2h namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: f49ae6ba-9b34-4cec-a9b7-537a3730a194 kind: Event lastTimestamp: "2026-04-24T21:37:47Z" message: Stopping container model-chainer-raw-7bb74 metadata: creationTimestamp: "2026-04-24T21:37:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:47Z" name: model-chainer-raw-7bb74-5899c94ff8-n7d2h.18a968b7fd9f21b2 namespace: kserve-ci-e2e-test resourceVersion: "14654" uid: 0f690df6-f857-415a-a490-ac5a9a0c0e1a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-55.ec2.internal source: component: kubelet host: ip-10-0-131-55.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:37:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-7bb74} kind: Pod name: model-chainer-raw-7bb74-5899c94ff8-n7d2h namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: f49ae6ba-9b34-4cec-a9b7-537a3730a194 kind: Event lastTimestamp: "2026-04-24T21:38:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:16Z" name: model-chainer-raw-7bb74-5899c94ff8-n7d2h.18a968b8edc42239 namespace: kserve-ci-e2e-test resourceVersion: "15030" uid: ee9e0821-b78a-45f3-bc93-0e0f2aa621c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-55.ec2.internal source: component: kubelet host: ip-10-0-131-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-7bb74-5899c94ff8 namespace: kserve-ci-e2e-test resourceVersion: "14527" uid: bef3e524-eb19-4803-b753-04410d998e77 kind: Event lastTimestamp: "2026-04-24T21:37:37Z" message: 'Created pod: model-chainer-raw-7bb74-5899c94ff8-n7d2h' metadata: creationTimestamp: "2026-04-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:37Z" name: model-chainer-raw-7bb74-5899c94ff8.18a968b5a75a3c86 namespace: kserve-ci-e2e-test resourceVersion: "14532" uid: a9fd406d-ae50-47d5-8ad2-308f10b5a8fb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-7bb74 namespace: kserve-ci-e2e-test resourceVersion: "14524" uid: 9263a8fb-182e-436e-87d6-46a5aa8b286a kind: Event lastTimestamp: "2026-04-24T21:37:37Z" message: Scaled up replica set model-chainer-raw-7bb74-5899c94ff8 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:37Z" name: model-chainer-raw-7bb74.18a968b5a5f17857 namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: f75fed3a-7296-4ed2-bf1c-f3a0b2401a05 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-7bb74 namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: 91fc4ba5-95e7-47a9-a9f5-134fcd9870de kind: Event lastTimestamp: "2026-04-24T21:37:37Z" message: InferenceGraph [model-chainer-raw-7bb74] is Ready metadata: creationTimestamp: "2026-04-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:37:37Z" name: model-chainer-raw-7bb74.18a968b5ad1b4abe namespace: kserve-ci-e2e-test resourceVersion: "14544" uid: fb6bc328-1256-4428-8e62-60f082138235 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:39:17.661282Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-21211-64b57f5b4b-hc8gb namespace: kserve-ci-e2e-test resourceVersion: "15505" uid: 86bf6e33-9a0f-4a7b-8f46-856d8cc89f5b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-21211-64b57f5b4b-hc8gb to ip-10-0-131-55.ec2.internal metadata: creationTimestamp: "2026-04-24T21:39:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:39:17Z" name: model-chainer-raw-hpa-21211-64b57f5b4b-hc8gb.18a968cd008e6a7e namespace: kserve-ci-e2e-test resourceVersion: "15511" uid: b9a70c40-a1c0-4705-87ad-7f88e301e4ed reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6f5f9cfdf5-qjgtv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:17Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-21211-64b57f5b4b-hc8gb namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: 86bf6e33-9a0f-4a7b-8f46-856d8cc89f5b kind: Event lastTimestamp: "2026-04-24T21:39:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-21211-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:17Z" name: model-chainer-raw-hpa-21211-64b57f5b4b-hc8gb.18a968cd08cd07c7 namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: 4b854c85-9915-4b96-bffb-f492cb4aa6fd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-55.ec2.internal source: component: kubelet host: ip-10-0-131-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:18Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-21211-64b57f5b4b-hc8gb namespace: kserve-ci-e2e-test resourceVersion: "15518" uid: 86bf6e33-9a0f-4a7b-8f46-856d8cc89f5b kind: Event lastTimestamp: "2026-04-24T21:39:18Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:39:18Z" name: model-chainer-raw-hpa-21211-64b57f5b4b-hc8gb.18a968cd3d98c682 namespace: kserve-ci-e2e-test resourceVersion: "15534" uid: f282bb8e-6056-4770-a31c-48fd677c70b6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-21211} kind: Pod name: model-chainer-raw-hpa-21211-64b57f5b4b-hc8gb namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: 86bf6e33-9a0f-4a7b-8f46-856d8cc89f5b kind: Event lastTimestamp: "2026-04-24T21:39:18Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b42fc184cd9997665c45700486ec1c9301c424ede94aa53d528583b0fe44a7ad" already present on machine metadata: creationTimestamp: "2026-04-24T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:18Z" name: model-chainer-raw-hpa-21211-64b57f5b4b-hc8gb.18a968cd3efecce8 namespace: kserve-ci-e2e-test resourceVersion: "15536" uid: 6cf68027-994d-4e88-bb64-4e0bf827104c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-55.ec2.internal source: component: kubelet host: ip-10-0-131-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-21211} kind: Pod name: model-chainer-raw-hpa-21211-64b57f5b4b-hc8gb namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: 86bf6e33-9a0f-4a7b-8f46-856d8cc89f5b kind: Event lastTimestamp: "2026-04-24T21:39:18Z" message: 'Created container: model-chainer-raw-hpa-21211' metadata: creationTimestamp: "2026-04-24T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:18Z" name: model-chainer-raw-hpa-21211-64b57f5b4b-hc8gb.18a968cd41e9288c namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: f95e9614-db47-4576-81b2-cba6d6328317 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-55.ec2.internal source: component: kubelet host: ip-10-0-131-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-21211} kind: Pod name: model-chainer-raw-hpa-21211-64b57f5b4b-hc8gb namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: 86bf6e33-9a0f-4a7b-8f46-856d8cc89f5b kind: Event lastTimestamp: "2026-04-24T21:39:18Z" message: Started container model-chainer-raw-hpa-21211 metadata: creationTimestamp: "2026-04-24T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:18Z" name: model-chainer-raw-hpa-21211-64b57f5b4b-hc8gb.18a968cd4258f112 namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: 503135bd-23ae-4054-bbdd-017be2b093a7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-55.ec2.internal source: component: kubelet host: ip-10-0-131-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-21211} kind: Pod name: model-chainer-raw-hpa-21211-64b57f5b4b-hc8gb namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: 86bf6e33-9a0f-4a7b-8f46-856d8cc89f5b kind: Event lastTimestamp: "2026-04-24T21:39:27Z" message: Stopping container model-chainer-raw-hpa-21211 metadata: creationTimestamp: "2026-04-24T21:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:27Z" name: model-chainer-raw-hpa-21211-64b57f5b4b-hc8gb.18a968cf590ce513 namespace: kserve-ci-e2e-test resourceVersion: "15624" uid: 8f55e64f-be6a-4775-9818-b83939a23338 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-55.ec2.internal source: component: kubelet host: ip-10-0-131-55.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:39:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-21211} kind: Pod name: model-chainer-raw-hpa-21211-64b57f5b4b-hc8gb namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: 86bf6e33-9a0f-4a7b-8f46-856d8cc89f5b kind: Event lastTimestamp: "2026-04-24T21:39:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:39:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:55Z" name: model-chainer-raw-hpa-21211-64b57f5b4b-hc8gb.18a968d00d0bf175 namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: c0c11778-0d3a-45ac-9d52-dbe12bd9493f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-55.ec2.internal source: component: kubelet host: ip-10-0-131-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-21211-64b57f5b4b namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: 5372b378-ae6e-4aae-a32f-4be15d6ccbde kind: Event lastTimestamp: "2026-04-24T21:39:17Z" message: 'Created pod: model-chainer-raw-hpa-21211-64b57f5b4b-hc8gb' metadata: creationTimestamp: "2026-04-24T21:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:17Z" name: model-chainer-raw-hpa-21211-64b57f5b4b.18a968cd001e08be namespace: kserve-ci-e2e-test resourceVersion: "15506" uid: f8168245-f760-49ac-8f47-3343f4ade607 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-21211 namespace: kserve-ci-e2e-test resourceVersion: "15500" uid: 3ed0f162-a461-40e5-adfc-4b23db235376 kind: Event lastTimestamp: "2026-04-24T21:39:17Z" message: Scaled up replica set model-chainer-raw-hpa-21211-64b57f5b4b from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:17Z" name: model-chainer-raw-hpa-21211.18a968ccff55d1ec namespace: kserve-ci-e2e-test resourceVersion: "15504" uid: b087c6a8-82be-4231-b316-fae1e132dc5c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-21211 namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: dc62c895-fe57-476d-892d-c94a80ddd454 kind: Event lastTimestamp: "2026-04-24T21:39:17Z" message: InferenceGraph [model-chainer-raw-hpa-21211] is Ready metadata: creationTimestamp: "2026-04-24T21:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:17Z" name: model-chainer-raw-hpa-21211.18a968cd03700d0b namespace: kserve-ci-e2e-test resourceVersion: "15522" uid: 09d56ac0-49af-4894-964e-58d47374f362 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:45:31.980334Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44 namespace: kserve-ci-e2e-test resourceVersion: "18822" uid: 9163e3dd-d311-4173-ae91-0ca8da093322 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-3a3ae-predictor-7769cb5696-vds44 to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-24T21:45:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:45:31Z" name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44.18a9692427b5df0b namespace: kserve-ci-e2e-test resourceVersion: "18826" uid: 722133cb-141b-4854-a7e5-7406f4a6ebfc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6f5f9cfdf5-qjgtv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:32Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44 namespace: kserve-ci-e2e-test resourceVersion: "18831" uid: 9163e3dd-d311-4173-ae91-0ca8da093322 kind: Event lastTimestamp: "2026-04-24T21:45:32Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:45:32Z" name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44.18a96924406d2df3 namespace: kserve-ci-e2e-test resourceVersion: "18841" uid: a6631117-2a0b-4bbb-bfe8-a0670c309959 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44 namespace: kserve-ci-e2e-test resourceVersion: "18824" uid: 9163e3dd-d311-4173-ae91-0ca8da093322 kind: Event lastTimestamp: "2026-04-24T21:45:32Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b2a48068363ac4e806775467ef2dc5b6c858faecf49ae1f8bafc4262d036f97e" already present on machine metadata: creationTimestamp: "2026-04-24T21:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:32Z" name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44.18a9692441cdc9ca namespace: kserve-ci-e2e-test resourceVersion: "18843" uid: 67dc5a2d-3c8c-4b6b-8a53-800d071bafd5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44 namespace: kserve-ci-e2e-test resourceVersion: "18824" uid: 9163e3dd-d311-4173-ae91-0ca8da093322 kind: Event lastTimestamp: "2026-04-24T21:45:32Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:32Z" name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44.18a9692444946b0f namespace: kserve-ci-e2e-test resourceVersion: "18844" uid: b689624a-1b3a-4214-b28b-c7f52b8273fc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44 namespace: kserve-ci-e2e-test resourceVersion: "18824" uid: 9163e3dd-d311-4173-ae91-0ca8da093322 kind: Event lastTimestamp: "2026-04-24T21:45:32Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:32Z" name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44.18a96924450ab586 namespace: kserve-ci-e2e-test resourceVersion: "18846" uid: 411943a8-168d-4b10-966c-d255531c4e22 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44 namespace: kserve-ci-e2e-test resourceVersion: "18824" uid: 9163e3dd-d311-4173-ae91-0ca8da093322 kind: Event lastTimestamp: "2026-04-24T21:45:36Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:36Z" name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44.18a96925262762a5 namespace: kserve-ci-e2e-test resourceVersion: "18879" uid: dfe50175-6b29-4f7f-97e4-c71f3437ad39 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44 namespace: kserve-ci-e2e-test resourceVersion: "18824" uid: 9163e3dd-d311-4173-ae91-0ca8da093322 kind: Event lastTimestamp: "2026-04-24T21:45:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:36Z" name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44.18a9692529023527 namespace: kserve-ci-e2e-test resourceVersion: "18881" uid: c0e3f0e9-4f3b-410e-b43a-68b8f90f6ab8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44 namespace: kserve-ci-e2e-test resourceVersion: "18824" uid: 9163e3dd-d311-4173-ae91-0ca8da093322 kind: Event lastTimestamp: "2026-04-24T21:45:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:36Z" name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44.18a969252989398f namespace: kserve-ci-e2e-test resourceVersion: "18884" uid: e1195915-8e49-4e9f-8545-5b4c11905baa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44 namespace: kserve-ci-e2e-test resourceVersion: "18824" uid: 9163e3dd-d311-4173-ae91-0ca8da093322 kind: Event lastTimestamp: "2026-04-24T21:45:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:36Z" name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44.18a9692529938251 namespace: kserve-ci-e2e-test resourceVersion: "18885" uid: 5cbcefb1-83ad-4221-b299-70ada6bad42c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44 namespace: kserve-ci-e2e-test resourceVersion: "18824" uid: 9163e3dd-d311-4173-ae91-0ca8da093322 kind: Event lastTimestamp: "2026-04-24T21:45:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:36Z" name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44.18a969252c2d43c9 namespace: kserve-ci-e2e-test resourceVersion: "18888" uid: 1772159d-540b-434c-8289-7a2feb12c5cf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44 namespace: kserve-ci-e2e-test resourceVersion: "18824" uid: 9163e3dd-d311-4173-ae91-0ca8da093322 kind: Event lastTimestamp: "2026-04-24T21:45:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:36Z" name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44.18a969252cafe067 namespace: kserve-ci-e2e-test resourceVersion: "18889" uid: d79a8f2d-d083-49c5-8505-97cf70fa6090 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:45:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44 namespace: kserve-ci-e2e-test resourceVersion: "18824" uid: 9163e3dd-d311-4173-ae91-0ca8da093322 kind: Event lastTimestamp: "2026-04-24T21:46:54Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:54Z" name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44.18a969259ddb5238 namespace: kserve-ci-e2e-test resourceVersion: "19516" uid: 000f523b-9de7-4dc0-b454-b38cbfc2d6d4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44 namespace: kserve-ci-e2e-test resourceVersion: "18824" uid: 9163e3dd-d311-4173-ae91-0ca8da093322 kind: Event lastTimestamp: "2026-04-24T21:46:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:52Z" name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44.18a96936ce98acb3 namespace: kserve-ci-e2e-test resourceVersion: "19456" uid: 1c00da10-8668-41d5-9586-51a1bd80e5ee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44 namespace: kserve-ci-e2e-test resourceVersion: "18824" uid: 9163e3dd-d311-4173-ae91-0ca8da093322 kind: Event lastTimestamp: "2026-04-24T21:46:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:52Z" name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44.18a96936ce993c55 namespace: kserve-ci-e2e-test resourceVersion: "19458" uid: d5aa55a5-f0ad-47c4-9340-44632de61bce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44 namespace: kserve-ci-e2e-test resourceVersion: "18824" uid: 9163e3dd-d311-4173-ae91-0ca8da093322 kind: Event lastTimestamp: "2026-04-24T21:46:54Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:54Z" name: raw-sklearn-3a3ae-predictor-7769cb5696-vds44.18a969374ff2ebc3 namespace: kserve-ci-e2e-test resourceVersion: "19515" uid: 69004b27-73c8-4c6f-9baf-dde5793acbda reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-3a3ae-predictor-7769cb5696 namespace: kserve-ci-e2e-test resourceVersion: "18811" uid: ffa6262a-17cd-46e6-a4a8-80aa84f58225 kind: Event lastTimestamp: "2026-04-24T21:45:31Z" message: 'Created pod: raw-sklearn-3a3ae-predictor-7769cb5696-vds44' metadata: creationTimestamp: "2026-04-24T21:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:31Z" name: raw-sklearn-3a3ae-predictor-7769cb5696.18a9692427441ea1 namespace: kserve-ci-e2e-test resourceVersion: "18823" uid: 97bbcb21-b9c5-4f01-b893-74754a385926 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-3a3ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "18809" uid: 6e8c45d6-6cbc-4ec1-8763-19d7d8745369 kind: Event lastTimestamp: "2026-04-24T21:45:31Z" message: Scaled up replica set raw-sklearn-3a3ae-predictor-7769cb5696 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:31Z" name: raw-sklearn-3a3ae-predictor.18a96924249c9fbe namespace: kserve-ci-e2e-test resourceVersion: "18813" uid: 6973eb88-1b01-44c9-914c-7aed21389e3e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-3a3ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "18810" uid: 70c847e2-f859-4888-8b5a-208ec82343ea kind: Event lastTimestamp: "2026-04-24T21:45:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:46Z" name: raw-sklearn-3a3ae-predictor.18a96927a39e13ed namespace: kserve-ci-e2e-test resourceVersion: "18969" uid: 03e2beb8-2021-4cf7-92e0-306637796374 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-3a3ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "18810" uid: 70c847e2-f859-4888-8b5a-208ec82343ea kind: Event lastTimestamp: "2026-04-24T21:45:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:46Z" name: raw-sklearn-3a3ae-predictor.18a96927a40d3daf namespace: kserve-ci-e2e-test resourceVersion: "18971" uid: acfecf0a-ec89-4c3b-99c2-c0aae04e14b7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:46:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-3a3ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "18970" uid: 70c847e2-f859-4888-8b5a-208ec82343ea kind: Event lastTimestamp: "2026-04-24T21:46:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:47Z" name: raw-sklearn-3a3ae-predictor.18a9692b22cbbd20 namespace: kserve-ci-e2e-test resourceVersion: "19391" uid: 5cfd2601-a73f-43ac-8a69-c45aebeebff4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:46:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-3a3ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "18970" uid: 70c847e2-f859-4888-8b5a-208ec82343ea kind: Event lastTimestamp: "2026-04-24T21:46:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:47Z" name: raw-sklearn-3a3ae-predictor.18a9692b2336403f namespace: kserve-ci-e2e-test resourceVersion: "19392" uid: 5ec83b0e-f2b3-413f-9310-130d41497e04 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T21:45:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-3a3ae namespace: kserve-ci-e2e-test resourceVersion: "18798" uid: 27d476ef-130f-46c4-bdd3-7065e8ecbaca kind: Event lastTimestamp: "2026-04-24T21:46:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:46:44Z" name: raw-sklearn-3a3ae.18a96924247c2d65 namespace: kserve-ci-e2e-test resourceVersion: "19372" uid: 716987b3-3752-4894-8e60-917fbc51ade9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-3a3ae namespace: kserve-ci-e2e-test resourceVersion: "18798" uid: 27d476ef-130f-46c4-bdd3-7065e8ecbaca kind: Event lastTimestamp: "2026-04-24T21:45:31Z" message: 'Failed to update status for InferenceService "raw-sklearn-3a3ae": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-3a3ae": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:31Z" name: raw-sklearn-3a3ae.18a9692424fdf52e namespace: kserve-ci-e2e-test resourceVersion: "18815" uid: 44b0e87f-7a7d-42f5-80df-a55b1f0c1051 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-3a3ae namespace: kserve-ci-e2e-test resourceVersion: "19370" uid: 27d476ef-130f-46c4-bdd3-7065e8ecbaca kind: Event lastTimestamp: "2026-04-24T21:46:44Z" message: InferenceService [raw-sklearn-3a3ae] is Ready metadata: creationTimestamp: "2026-04-24T21:46:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:46:44Z" name: raw-sklearn-3a3ae.18a9693501534312 namespace: kserve-ci-e2e-test resourceVersion: "19371" uid: 6649f5ed-5a28-4747-9982-c16f4cc41dfa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:46:52.174593Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2 namespace: kserve-ci-e2e-test resourceVersion: "19472" uid: f621ee3d-e2a2-4f8a-85f4-155e3135575c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2 to ip-10-0-139-184.ec2.internal metadata: creationTimestamp: "2026-04-24T21:46:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:46:52Z" name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2.18a96936d3a9330d namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: 9e257cd2-fb84-45d6-b721-888b61e3d254 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6f5f9cfdf5-qjgtv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:52Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: f621ee3d-e2a2-4f8a-85f4-155e3135575c kind: Event lastTimestamp: "2026-04-24T21:46:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-runtime-fba15-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:52Z" name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2.18a96936e0f2c186 namespace: kserve-ci-e2e-test resourceVersion: "19487" uid: 5b88f847-a7de-45ec-a9e4-600daded5667 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:53Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2 namespace: kserve-ci-e2e-test resourceVersion: "19482" uid: f621ee3d-e2a2-4f8a-85f4-155e3135575c kind: Event lastTimestamp: "2026-04-24T21:46:53Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:46:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:46:53Z" name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2.18a9693710f31b5f namespace: kserve-ci-e2e-test resourceVersion: "19496" uid: a469f702-3a45-43e6-9fb4-6df11e96d225 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: f621ee3d-e2a2-4f8a-85f4-155e3135575c kind: Event lastTimestamp: "2026-04-24T21:46:53Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b2a48068363ac4e806775467ef2dc5b6c858faecf49ae1f8bafc4262d036f97e" already present on machine metadata: creationTimestamp: "2026-04-24T21:46:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:53Z" name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2.18a96937124cb062 namespace: kserve-ci-e2e-test resourceVersion: "19498" uid: 61d01ee9-df94-4ae6-b43b-7f8ffe345d26 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: f621ee3d-e2a2-4f8a-85f4-155e3135575c kind: Event lastTimestamp: "2026-04-24T21:46:53Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:46:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:53Z" name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2.18a969371571b714 namespace: kserve-ci-e2e-test resourceVersion: "19502" uid: 59654d59-e165-4052-a9bf-1115938bcc02 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: f621ee3d-e2a2-4f8a-85f4-155e3135575c kind: Event lastTimestamp: "2026-04-24T21:46:53Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:46:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:53Z" name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2.18a9693715e0a454 namespace: kserve-ci-e2e-test resourceVersion: "19503" uid: 81b209af-64bc-4489-9793-5224753809f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: f621ee3d-e2a2-4f8a-85f4-155e3135575c kind: Event lastTimestamp: "2026-04-24T21:46:57Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:46:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:57Z" name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2.18a96938109febd7 namespace: kserve-ci-e2e-test resourceVersion: "19544" uid: 03aa411b-ee90-431e-a564-ef8d476da844 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: f621ee3d-e2a2-4f8a-85f4-155e3135575c kind: Event lastTimestamp: "2026-04-24T21:46:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:46:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:57Z" name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2.18a96938138b7f2d namespace: kserve-ci-e2e-test resourceVersion: "19546" uid: 09717369-f9f5-40ba-bbb4-d5014f12ed68 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: f621ee3d-e2a2-4f8a-85f4-155e3135575c kind: Event lastTimestamp: "2026-04-24T21:46:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:46:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:57Z" name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2.18a9693813fbbcc4 namespace: kserve-ci-e2e-test resourceVersion: "19547" uid: 2f91406d-d21c-43ac-a8ab-66d3cdc7c3cc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: f621ee3d-e2a2-4f8a-85f4-155e3135575c kind: Event lastTimestamp: "2026-04-24T21:46:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:46:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:57Z" name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2.18a9693814062643 namespace: kserve-ci-e2e-test resourceVersion: "19550" uid: 86cfe04e-0781-4e2b-8902-dbe93367d767 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: f621ee3d-e2a2-4f8a-85f4-155e3135575c kind: Event lastTimestamp: "2026-04-24T21:46:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:46:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:57Z" name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2.18a9693816ad8089 namespace: kserve-ci-e2e-test resourceVersion: "19551" uid: 48790ebb-b605-4b48-b310-ee99d863723f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: f621ee3d-e2a2-4f8a-85f4-155e3135575c kind: Event lastTimestamp: "2026-04-24T21:46:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:46:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:57Z" name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2.18a96938171aa5ec namespace: kserve-ci-e2e-test resourceVersion: "19552" uid: acc55277-e39b-4aa0-b098-14f36a82d732 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:46:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: f621ee3d-e2a2-4f8a-85f4-155e3135575c kind: Event lastTimestamp: "2026-04-24T21:48:15Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:15Z" name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2.18a969388857d304 namespace: kserve-ci-e2e-test resourceVersion: "20168" uid: 5b5b03c6-648b-4c9a-a17d-0ae02e866009 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: f621ee3d-e2a2-4f8a-85f4-155e3135575c kind: Event lastTimestamp: "2026-04-24T21:48:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:12Z" name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2.18a969497947bdeb namespace: kserve-ci-e2e-test resourceVersion: "20110" uid: 68188439-a254-4523-b9a0-54e93ae70e91 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: f621ee3d-e2a2-4f8a-85f4-155e3135575c kind: Event lastTimestamp: "2026-04-24T21:48:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:12Z" name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2.18a9694979485c7b namespace: kserve-ci-e2e-test resourceVersion: "20111" uid: cab1b079-8915-43a5-9d14-bf3b5e5aaa0e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: f621ee3d-e2a2-4f8a-85f4-155e3135575c kind: Event lastTimestamp: "2026-04-24T21:48:15Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:15Z" name: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2.18a9694a3a726985 namespace: kserve-ci-e2e-test resourceVersion: "20167" uid: 4dec9fd6-7fe6-4796-ad18-52c8156de90b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-184.ec2.internal source: component: kubelet host: ip-10-0-139-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-fba15-predictor-5985f556c8 namespace: kserve-ci-e2e-test resourceVersion: "19463" uid: 26a325ff-4694-43b9-9981-e172c8c149b6 kind: Event lastTimestamp: "2026-04-24T21:46:52Z" message: 'Created pod: raw-sklearn-runtime-fba15-predictor-5985f556c8-fcbb2' metadata: creationTimestamp: "2026-04-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:52Z" name: raw-sklearn-runtime-fba15-predictor-5985f556c8.18a96936d3370178 namespace: kserve-ci-e2e-test resourceVersion: "19475" uid: 1add96aa-ef8c-45e9-a2ab-cbdd38fe4815 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-fba15-predictor namespace: kserve-ci-e2e-test resourceVersion: "19461" uid: fd833230-f999-41aa-91af-d2cc3205b31f kind: Event lastTimestamp: "2026-04-24T21:46:52Z" message: Scaled up replica set raw-sklearn-runtime-fba15-predictor-5985f556c8 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:52Z" name: raw-sklearn-runtime-fba15-predictor.18a96936d04b22a7 namespace: kserve-ci-e2e-test resourceVersion: "19465" uid: 34d6e0e9-e7e8-45b1-ac64-3bdd4c6f0394 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:47:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-fba15-predictor namespace: kserve-ci-e2e-test resourceVersion: "19462" uid: 15967b3a-cfdc-4ae8-9e86-31e61a34840d kind: Event lastTimestamp: "2026-04-24T21:47:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:47:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:22Z" name: raw-sklearn-runtime-fba15-predictor.18a9693a4ed6c080 namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 1010ce29-8ba9-47c7-aaf9-b5642ef459f3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:47:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-fba15-predictor namespace: kserve-ci-e2e-test resourceVersion: "19462" uid: 15967b3a-cfdc-4ae8-9e86-31e61a34840d kind: Event lastTimestamp: "2026-04-24T21:47:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:47:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:22Z" name: raw-sklearn-runtime-fba15-predictor.18a9693a4f3dd5b7 namespace: kserve-ci-e2e-test resourceVersion: "19732" uid: 67d0ad0d-2a8b-40f4-b096-2c06995360ba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:47:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-fba15-predictor namespace: kserve-ci-e2e-test resourceVersion: "19621" uid: 15967b3a-cfdc-4ae8-9e86-31e61a34840d kind: Event lastTimestamp: "2026-04-24T21:48:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:07Z" name: raw-sklearn-runtime-fba15-predictor.18a969414d4819dc namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: 58dcfff4-01d9-4aba-9816-2288c54f1634 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:47:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-fba15-predictor namespace: kserve-ci-e2e-test resourceVersion: "19621" uid: 15967b3a-cfdc-4ae8-9e86-31e61a34840d kind: Event lastTimestamp: "2026-04-24T21:48:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:07Z" name: raw-sklearn-runtime-fba15-predictor.18a969414dba87f4 namespace: kserve-ci-e2e-test resourceVersion: "20050" uid: fea164fc-49b2-4e52-9c88-5da5315e7c97 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T21:46:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-fba15 namespace: kserve-ci-e2e-test resourceVersion: "19432" uid: 6411407a-c739-4a12-acd3-4e2d4c8917bc kind: Event lastTimestamp: "2026-04-24T21:48:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:48:05Z" name: raw-sklearn-runtime-fba15.18a96936d028f309 namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: cb5d48bd-5375-4779-9b7a-62a36566caa1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-fba15 namespace: kserve-ci-e2e-test resourceVersion: "19432" uid: 6411407a-c739-4a12-acd3-4e2d4c8917bc kind: Event lastTimestamp: "2026-04-24T21:46:52Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-fba15": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-fba15": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:46:52Z" name: raw-sklearn-runtime-fba15.18a96936d09a6683 namespace: kserve-ci-e2e-test resourceVersion: "19467" uid: 211f2701-0647-437a-8a8b-e81c0a226428 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-fba15 namespace: kserve-ci-e2e-test resourceVersion: "20032" uid: 6411407a-c739-4a12-acd3-4e2d4c8917bc kind: Event lastTimestamp: "2026-04-24T21:48:05Z" message: InferenceService [raw-sklearn-runtime-fba15] is Ready metadata: creationTimestamp: "2026-04-24T21:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:48:05Z" name: raw-sklearn-runtime-fba15.18a96947ebe1af0e namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 102caf85-ceb8-4945-be2f-391359010bbc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "20314"