--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:25:28.817056Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-b713a0-predictor-77df54fb5f-5nrvb namespace: kserve-ci-e2e-test resourceVersion: "20385" uid: 625afa52-f752-4d0a-9a4b-318a66355400 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-b713a0-predictor-77df54fb5f-5nrvb to ip-10-0-132-145.ec2.internal metadata: creationTimestamp: "2026-07-01T18:25:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:25:28Z" name: isvc-init-fail-b713a0-predictor-77df54fb5f-5nrvb.18be3db00c694afd namespace: kserve-ci-e2e-test resourceVersion: "20391" uid: 50e4fdf9-b252-4523-b5db-07f4f048610b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-849956d989-wqhnw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:29Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-b713a0-predictor-77df54fb5f-5nrvb namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 625afa52-f752-4d0a-9a4b-318a66355400 kind: Event lastTimestamp: "2026-07-01T18:25:29Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:25:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:25:29Z" name: isvc-init-fail-b713a0-predictor-77df54fb5f-5nrvb.18be3db02645a956 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: 20cf3ecb-6392-4ca3-a97e-bc456cc211d5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:25:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-b713a0-predictor-77df54fb5f-5nrvb namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: 625afa52-f752-4d0a-9a4b-318a66355400 kind: Event lastTimestamp: "2026-07-01T18:25:33Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:25:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:33Z" name: isvc-init-fail-b713a0-predictor-77df54fb5f-5nrvb.18be3db027cc7d04 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: cee318b2-5a07-4ec3-b945-a7298f7ee37e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:25:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-b713a0-predictor-77df54fb5f-5nrvb namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: 625afa52-f752-4d0a-9a4b-318a66355400 kind: Event lastTimestamp: "2026-07-01T18:25:33Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:25:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:33Z" name: isvc-init-fail-b713a0-predictor-77df54fb5f-5nrvb.18be3db02b5f8a1e namespace: kserve-ci-e2e-test resourceVersion: "20449" uid: e2ed6e93-5882-45ab-ab85-3e24a0aa1e8f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:25:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-b713a0-predictor-77df54fb5f-5nrvb namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: 625afa52-f752-4d0a-9a4b-318a66355400 kind: Event lastTimestamp: "2026-07-01T18:25:33Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:25:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:33Z" name: isvc-init-fail-b713a0-predictor-77df54fb5f-5nrvb.18be3db02bdd7ee6 namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: cbc078e9-71be-4962-b8db-0cf8618bdff6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-b713a0-predictor-77df54fb5f-5nrvb namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: 625afa52-f752-4d0a-9a4b-318a66355400 kind: Event lastTimestamp: "2026-07-01T18:25:34Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-07-01T18:25:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:34Z" name: isvc-init-fail-b713a0-predictor-77df54fb5f-5nrvb.18be3db157cb2dc2 namespace: kserve-ci-e2e-test resourceVersion: "20479" uid: 6fe3902e-d409-4295-9ac4-c46798a9325f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-b713a0-predictor-77df54fb5f namespace: kserve-ci-e2e-test resourceVersion: "20378" uid: 7e35d04b-ad8f-4eb5-902b-332baf02c005 kind: Event lastTimestamp: "2026-07-01T18:25:28Z" message: 'Created pod: isvc-init-fail-b713a0-predictor-77df54fb5f-5nrvb' metadata: creationTimestamp: "2026-07-01T18:25:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:25:28Z" name: isvc-init-fail-b713a0-predictor-77df54fb5f.18be3db00bd36664 namespace: kserve-ci-e2e-test resourceVersion: "20386" uid: 2a2819e4-932f-4f6e-ac5f-ceffff4d62c6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-b713a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "20376" uid: 18f7c084-1be4-4706-ae95-31cd01382ffd kind: Event lastTimestamp: "2026-07-01T18:25:28Z" message: Scaled up replica set isvc-init-fail-b713a0-predictor-77df54fb5f from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:25:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:25:28Z" name: isvc-init-fail-b713a0-predictor.18be3db0090cad85 namespace: kserve-ci-e2e-test resourceVersion: "20379" uid: 64510e3a-07e2-40a2-9c32-7c4e78b4b9f2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T18:25:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-b713a0 namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: 1411bef3-a6b9-47ee-b534-fecd8e7a5d3f kind: Event lastTimestamp: "2026-07-01T18:25:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:25:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:25:33Z" name: isvc-init-fail-b713a0.18be3db009128d86 namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: ebb11148-39b6-4223-8893-eb66256f9a76 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-b713a0 namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: 1411bef3-a6b9-47ee-b534-fecd8e7a5d3f kind: Event lastTimestamp: "2026-07-01T18:25:28Z" message: 'Failed to update status for InferenceService "isvc-init-fail-b713a0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-b713a0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:25:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:25:28Z" name: isvc-init-fail-b713a0.18be3db009991800 namespace: kserve-ci-e2e-test resourceVersion: "20382" uid: 2894c792-ab9b-4e1d-b79b-f9a0d79eeb64 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-b713a0 namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: 1411bef3-a6b9-47ee-b534-fecd8e7a5d3f kind: Event lastTimestamp: "2026-07-01T18:25:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-b713a0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:25:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:25:28Z" name: isvc-init-fail-b713a0.18be3db009997d8e namespace: kserve-ci-e2e-test resourceVersion: "20383" uid: e32b24fc-f457-4432-b786-05818cfc40c1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:19:04.766031Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17273" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc to ip-10-0-132-145.ec2.internal metadata: creationTimestamp: "2026-07-01T18:19:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:19:04Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d56a12fb66a namespace: kserve-ci-e2e-test resourceVersion: "17277" uid: fe413a76-fc2a-423c-a028-21cdb71c6656 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-849956d989-wqhnw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: "2026-07-01T18:19:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-raw-dbad7-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:19:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:04Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d56ada1039b namespace: kserve-ci-e2e-test resourceVersion: "17286" uid: 857e6c31-5d41-4900-ad64-566510fe9be0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: "2026-07-01T18:19:05Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:19:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:19:05Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d56dee62e87 namespace: kserve-ci-e2e-test resourceVersion: "17296" uid: 27631ac4-ffb8-4a7f-a1d2-905dbc2d04ca reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: "2026-07-01T18:19:05Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:19:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:05Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d56e07209bb namespace: kserve-ci-e2e-test resourceVersion: "17298" uid: c5ee82a7-229f-4f24-aef8-12cee06e8c15 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: "2026-07-01T18:19:05Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:19:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:05Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d56e34956f1 namespace: kserve-ci-e2e-test resourceVersion: "17299" uid: 67962c83-3e4e-4b92-b99a-dd61fe9f7d34 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: "2026-07-01T18:19:05Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:19:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:05Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d56e3c81c84 namespace: kserve-ci-e2e-test resourceVersion: "17301" uid: 5a756429-48cc-4102-b000-f168c4dbb3d5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: "2026-07-01T18:19:09Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:19:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:09Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d57c237ad75 namespace: kserve-ci-e2e-test resourceVersion: "17338" uid: d647c0e4-36e3-4303-bd1c-f56e208c0dd3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: "2026-07-01T18:19:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:19:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:09Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d57c554de6c namespace: kserve-ci-e2e-test resourceVersion: "17340" uid: 1f38a049-e787-405c-9227-fdbc6d9a1f30 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: "2026-07-01T18:19:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:19:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:09Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d57c5cda19d namespace: kserve-ci-e2e-test resourceVersion: "17343" uid: 33357d75-5a91-469d-82a1-bf4d166fd22d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: "2026-07-01T18:19:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:19:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:09Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d57c5da29d5 namespace: kserve-ci-e2e-test resourceVersion: "17344" uid: 2fed7408-308f-4dff-bb42-8c86d6c0dd44 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: "2026-07-01T18:19:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:19:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:09Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d57c91dbba3 namespace: kserve-ci-e2e-test resourceVersion: "17346" uid: 74a843b8-33b3-40f0-b626-fdb814d2d9d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: "2026-07-01T18:19:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:19:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:09Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d57c999b9b9 namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 552b2634-21dd-4c03-98dd-2665b156d283 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: "2026-07-01T18:19:09Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:2fd41eb0f45027343da0a3e7ab10342a4e3f1970b88cf54fcc671e20be2ec985" already present on machine metadata: creationTimestamp: "2026-07-01T18:19:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:09Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d57c9a493bb namespace: kserve-ci-e2e-test resourceVersion: "17348" uid: a01bb2b1-dcc5-48ce-9ea0-a5cd2a0092c6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: "2026-07-01T18:19:09Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-01T18:19:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:09Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d57cda1d34c namespace: kserve-ci-e2e-test resourceVersion: "17350" uid: 0d8cb436-aa5b-488a-9fd7-e6e2b36d7dac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: "2026-07-01T18:19:09Z" message: Started container agent metadata: creationTimestamp: "2026-07-01T18:19:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:09Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d57ce23b51a namespace: kserve-ci-e2e-test resourceVersion: "17351" uid: f26da944-370b-468f-a9ed-84359512ed21 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T18:19:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: "2026-07-01T18:20:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:19:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:54Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d57fe517133 namespace: kserve-ci-e2e-test resourceVersion: "18234" uid: 9a3bd939-e694-4f86-8176-2e99135ac605 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T18:19:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: "2026-07-01T18:20:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:19:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:44Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d57fe5baee1 namespace: kserve-ci-e2e-test resourceVersion: "18141" uid: 1cb2493e-6022-45eb-a21d-72ab63d56f2e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: "2026-07-01T18:20:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:40Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d6cd30b5290 namespace: kserve-ci-e2e-test resourceVersion: "18067" uid: 6666be52-d75d-4fec-b56e-5915933737ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: "2026-07-01T18:20:40Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-01T18:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:40Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d6cd30be262 namespace: kserve-ci-e2e-test resourceVersion: "18073" uid: eae2a53b-449c-45a6-b844-3af4ca26c91e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: "2026-07-01T18:20:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:40Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d6cd30c2fa9 namespace: kserve-ci-e2e-test resourceVersion: "18077" uid: cf77f021-84eb-4b4a-b9ff-6671395c63ef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:20:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: d415cd5a-da86-48ed-8e9f-4666a61d5542 kind: Event lastTimestamp: "2026-07-01T18:20:51Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:20:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:51Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc.18be3d6d2e9485c8 namespace: kserve-ci-e2e-test resourceVersion: "18206" uid: 6bcfe4c0-13f5-4c23-b761-95c2c4a63cde reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-dbad7-predictor-f57b9b598 namespace: kserve-ci-e2e-test resourceVersion: "17262" uid: 08d39326-e0d6-4769-859e-40f0dc8cfed4 kind: Event lastTimestamp: "2026-07-01T18:19:04Z" message: 'Created pod: isvc-logger-raw-dbad7-predictor-f57b9b598-6fzwc' metadata: creationTimestamp: "2026-07-01T18:19:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:19:04Z" name: isvc-logger-raw-dbad7-predictor-f57b9b598.18be3d56a09cfd2b namespace: kserve-ci-e2e-test resourceVersion: "17274" uid: da42f3b7-5e28-4674-9e43-5e2ee4e57752 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-dbad7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17259" uid: 70d87e7d-029c-4e15-934d-03bdd72962d4 kind: Event lastTimestamp: "2026-07-01T18:19:04Z" message: Scaled up replica set isvc-logger-raw-dbad7-predictor-f57b9b598 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:19:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:19:04Z" name: isvc-logger-raw-dbad7-predictor.18be3d569cd19f08 namespace: kserve-ci-e2e-test resourceVersion: "17263" uid: cf23c5ca-ad59-4ee9-845d-3556d0145d33 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:19:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-dbad7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17260" uid: a7039a54-a361-4d88-a7af-4ff366017cf6 kind: Event lastTimestamp: "2026-07-01T18:19:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:19:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:19:49Z" name: isvc-logger-raw-dbad7-predictor.18be3d5a1b19150d namespace: kserve-ci-e2e-test resourceVersion: "17651" uid: e057cff0-64d4-4a89-bf5c-52b178039ecb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:19:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-dbad7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17260" uid: a7039a54-a361-4d88-a7af-4ff366017cf6 kind: Event lastTimestamp: "2026-07-01T18:19:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:19:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:19:49Z" name: isvc-logger-raw-dbad7-predictor.18be3d5a1b840071 namespace: kserve-ci-e2e-test resourceVersion: "17652" uid: 6c01c12f-a4bb-47e7-bf5f-eb332f0dd9f5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:19:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-dbad7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17436" uid: a7039a54-a361-4d88-a7af-4ff366017cf6 kind: Event lastTimestamp: "2026-07-01T18:20:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:19:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:34Z" name: isvc-logger-raw-dbad7-predictor.18be3d5d9a7092a3 namespace: kserve-ci-e2e-test resourceVersion: "17982" uid: 5725a8b0-2771-433b-8596-8a9cdd1ebe59 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:19:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-dbad7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17436" uid: a7039a54-a361-4d88-a7af-4ff366017cf6 kind: Event lastTimestamp: "2026-07-01T18:20:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:19:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:34Z" name: isvc-logger-raw-dbad7-predictor.18be3d5d9ae37c60 namespace: kserve-ci-e2e-test resourceVersion: "17983" uid: 46aa13cf-4b2d-406b-9057-6c33349b3459 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T18:19:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-dbad7 namespace: kserve-ci-e2e-test resourceVersion: "17251" uid: 884ba636-127e-49dd-83c5-d84d260a01c9 kind: Event lastTimestamp: "2026-07-01T18:20:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:19:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:20:25Z" name: isvc-logger-raw-dbad7.18be3d569cd8776a namespace: kserve-ci-e2e-test resourceVersion: "17907" uid: 2570e5ec-fbc4-4ce0-892e-63bd9e61302e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-dbad7 namespace: kserve-ci-e2e-test resourceVersion: "17251" uid: 884ba636-127e-49dd-83c5-d84d260a01c9 kind: Event lastTimestamp: "2026-07-01T18:19:04Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-dbad7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-dbad7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:19:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:19:04Z" name: isvc-logger-raw-dbad7.18be3d569d7db472 namespace: kserve-ci-e2e-test resourceVersion: "17267" uid: 6201768c-2f1e-4e70-937f-8c056adc5294 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-dbad7 namespace: kserve-ci-e2e-test resourceVersion: "17905" uid: 884ba636-127e-49dd-83c5-d84d260a01c9 kind: Event lastTimestamp: "2026-07-01T18:20:24Z" message: InferenceService [isvc-logger-raw-dbad7] is Ready metadata: creationTimestamp: "2026-07-01T18:20:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:20:24Z" name: isvc-logger-raw-dbad7.18be3d694cb08919 namespace: kserve-ci-e2e-test resourceVersion: "17906" uid: 5c1d807b-167d-47d7-9db8-7256f6d23658 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:23:35.420120Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt namespace: kserve-ci-e2e-test resourceVersion: "19414" uid: cddfaabf-62a3-4671-af4d-a393ccfc78f9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-72dc06-predictor-6cb65cb8db-9fztt to ip-10-0-132-145.ec2.internal metadata: creationTimestamp: "2026-07-01T18:23:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:23:35Z" name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt.18be3d95a56d66e3 namespace: kserve-ci-e2e-test resourceVersion: "19418" uid: 93713fb5-ac2e-4f3b-b2bf-4d7858c6acec reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-849956d989-wqhnw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:35Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt namespace: kserve-ci-e2e-test resourceVersion: "19423" uid: cddfaabf-62a3-4671-af4d-a393ccfc78f9 kind: Event lastTimestamp: "2026-07-01T18:23:35Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:23:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:23:35Z" name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt.18be3d95bf7ecc7c namespace: kserve-ci-e2e-test resourceVersion: "19433" uid: 2e22878f-e21e-4f31-8d73-9feea2b7638b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: cddfaabf-62a3-4671-af4d-a393ccfc78f9 kind: Event lastTimestamp: "2026-07-01T18:23:35Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:23:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:35Z" name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt.18be3d95c0d7d33f namespace: kserve-ci-e2e-test resourceVersion: "19436" uid: 7992e348-8c0e-4607-ab55-9ea41e99917c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: cddfaabf-62a3-4671-af4d-a393ccfc78f9 kind: Event lastTimestamp: "2026-07-01T18:23:35Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:23:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:35Z" name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt.18be3d95c424e56e namespace: kserve-ci-e2e-test resourceVersion: "19439" uid: c51d37ae-497e-4938-9eca-399b87b558dd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: cddfaabf-62a3-4671-af4d-a393ccfc78f9 kind: Event lastTimestamp: "2026-07-01T18:23:35Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:23:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:35Z" name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt.18be3d95c4a8d28b namespace: kserve-ci-e2e-test resourceVersion: "19440" uid: 79bd4f97-66e9-4555-904f-92e0d087e16d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: cddfaabf-62a3-4671-af4d-a393ccfc78f9 kind: Event lastTimestamp: "2026-07-01T18:23:39Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:39Z" name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt.18be3d96adb13592 namespace: kserve-ci-e2e-test resourceVersion: "19470" uid: 073cc829-d7e0-475d-b0db-306b40f4d0f5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: cddfaabf-62a3-4671-af4d-a393ccfc78f9 kind: Event lastTimestamp: "2026-07-01T18:23:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:39Z" name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt.18be3d96b1273e6e namespace: kserve-ci-e2e-test resourceVersion: "19474" uid: 2ce1a55c-d3b6-4a04-88db-dc212fd6a031 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: cddfaabf-62a3-4671-af4d-a393ccfc78f9 kind: Event lastTimestamp: "2026-07-01T18:23:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:39Z" name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt.18be3d96b1a43458 namespace: kserve-ci-e2e-test resourceVersion: "19477" uid: 7f548c28-b59d-45ee-8c07-7f925706fe37 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: cddfaabf-62a3-4671-af4d-a393ccfc78f9 kind: Event lastTimestamp: "2026-07-01T18:23:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:39Z" name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt.18be3d96b1afab04 namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: 359ccea9-0b2f-433f-b668-7cf6a9ac4618 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: cddfaabf-62a3-4671-af4d-a393ccfc78f9 kind: Event lastTimestamp: "2026-07-01T18:23:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:39Z" name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt.18be3d96b4c316a7 namespace: kserve-ci-e2e-test resourceVersion: "19480" uid: da3d52d3-5e73-4fca-8cdd-cd8c46c949ef reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: cddfaabf-62a3-4671-af4d-a393ccfc78f9 kind: Event lastTimestamp: "2026-07-01T18:23:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:39Z" name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt.18be3d96b5416c23 namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: 6ab34936-d9c0-4a18-918a-21a6a75f7620 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T18:23:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: cddfaabf-62a3-4671-af4d-a393ccfc78f9 kind: Event lastTimestamp: "2026-07-01T18:25:26Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:23:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:26Z" name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt.18be3d96e9f13bea namespace: kserve-ci-e2e-test resourceVersion: "20348" uid: 1c68fc78-8cab-4ffe-a26b-ef8c36e4c836 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: cddfaabf-62a3-4671-af4d-a393ccfc78f9 kind: Event lastTimestamp: "2026-07-01T18:25:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:25:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:23Z" name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt.18be3daed98821a0 namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: 52524fae-290b-402f-ae75-01c5cdfef08e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: cddfaabf-62a3-4671-af4d-a393ccfc78f9 kind: Event lastTimestamp: "2026-07-01T18:25:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:25:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:23Z" name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt.18be3daed988ca6d namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: 1ddb3717-5484-4daa-bb47-5d8a6edbcb2a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: cddfaabf-62a3-4671-af4d-a393ccfc78f9 kind: Event lastTimestamp: "2026-07-01T18:25:26Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:26Z" name: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt.18be3daf9858bb97 namespace: kserve-ci-e2e-test resourceVersion: "20347" uid: b0af1af3-ce84-469f-9ccd-d78ba7022efe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-72dc06-predictor-6cb65cb8db namespace: kserve-ci-e2e-test resourceVersion: "19405" uid: c80decd8-7590-4eea-a9db-cd449b2f1f2a kind: Event lastTimestamp: "2026-07-01T18:23:35Z" message: 'Created pod: isvc-primary-72dc06-predictor-6cb65cb8db-9fztt' metadata: creationTimestamp: "2026-07-01T18:23:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:23:35Z" name: isvc-primary-72dc06-predictor-6cb65cb8db.18be3d95a4debb71 namespace: kserve-ci-e2e-test resourceVersion: "19415" uid: 2a7a3257-3099-4b9a-a6c7-990762b91548 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-72dc06-predictor namespace: kserve-ci-e2e-test resourceVersion: "19403" uid: 0367b16f-a69b-4af9-9321-0fde419c6297 kind: Event lastTimestamp: "2026-07-01T18:23:35Z" message: Scaled up replica set isvc-primary-72dc06-predictor-6cb65cb8db from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:23:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:23:35Z" name: isvc-primary-72dc06-predictor.18be3d95a00939ab namespace: kserve-ci-e2e-test resourceVersion: "19406" uid: ce614748-4cfc-492a-9b8c-0ae844453055 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-72dc06-predictor namespace: kserve-ci-e2e-test resourceVersion: "19404" uid: 6747cd66-1f53-4657-b4a6-a8ac0c9b4517 kind: Event lastTimestamp: "2026-07-01T18:23:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:23:50Z" name: isvc-primary-72dc06-predictor.18be3d991ed39742 namespace: kserve-ci-e2e-test resourceVersion: "19555" uid: dfd30530-0f85-4c4a-8143-34fda91508c8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-72dc06-predictor namespace: kserve-ci-e2e-test resourceVersion: "19404" uid: 6747cd66-1f53-4657-b4a6-a8ac0c9b4517 kind: Event lastTimestamp: "2026-07-01T18:23:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:23:50Z" name: isvc-primary-72dc06-predictor.18be3d991f3390ee namespace: kserve-ci-e2e-test resourceVersion: "19557" uid: f9a282f4-06e9-45b8-8360-9a911ef28e7b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:24:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-72dc06-predictor namespace: kserve-ci-e2e-test resourceVersion: "19556" uid: 6747cd66-1f53-4657-b4a6-a8ac0c9b4517 kind: Event lastTimestamp: "2026-07-01T18:25:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:25:20Z" name: isvc-primary-72dc06-predictor.18be3d9c9e610d1a namespace: kserve-ci-e2e-test resourceVersion: "20264" uid: 2dcd4cab-0fe1-49c4-bae8-ea05b1fc7883 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:24:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-72dc06-predictor namespace: kserve-ci-e2e-test resourceVersion: "19556" uid: 6747cd66-1f53-4657-b4a6-a8ac0c9b4517 kind: Event lastTimestamp: "2026-07-01T18:25:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:25:20Z" name: isvc-primary-72dc06-predictor.18be3d9c9edc2c80 namespace: kserve-ci-e2e-test resourceVersion: "20265" uid: bbf7e09a-1f17-4e3d-b29e-541b393ad38f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T18:23:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-72dc06 namespace: kserve-ci-e2e-test resourceVersion: "19396" uid: 0868369f-50c6-4899-a5ab-c242bda9d300 kind: Event lastTimestamp: "2026-07-01T18:24:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:23:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:24:57Z" name: isvc-primary-72dc06.18be3d95a00d6042 namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: e565b6d4-5879-4bda-b004-3da751740c97 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-72dc06 namespace: kserve-ci-e2e-test resourceVersion: "19396" uid: 0868369f-50c6-4899-a5ab-c242bda9d300 kind: Event lastTimestamp: "2026-07-01T18:23:35Z" message: 'Failed to update status for InferenceService "isvc-primary-72dc06": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-72dc06": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:23:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:23:35Z" name: isvc-primary-72dc06.18be3d95a08d97b0 namespace: kserve-ci-e2e-test resourceVersion: "19409" uid: 7f1ba3e7-4e4b-4f4f-8be8-8c2049f787ea reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-72dc06 namespace: kserve-ci-e2e-test resourceVersion: "19396" uid: 0868369f-50c6-4899-a5ab-c242bda9d300 kind: Event lastTimestamp: "2026-07-01T18:23:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-72dc06": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:23:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:23:35Z" name: isvc-primary-72dc06.18be3d95a08e0216 namespace: kserve-ci-e2e-test resourceVersion: "19410" uid: baada4b5-0063-4c91-840a-7a2513a8a887 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:24:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-72dc06 namespace: kserve-ci-e2e-test resourceVersion: "20032" uid: 0868369f-50c6-4899-a5ab-c242bda9d300 kind: Event lastTimestamp: "2026-07-01T18:24:56Z" message: InferenceService [isvc-primary-72dc06] is Ready metadata: creationTimestamp: "2026-07-01T18:24:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:24:56Z" name: isvc-primary-72dc06.18be3da8a1cb66cb namespace: kserve-ci-e2e-test resourceVersion: "20034" uid: 751a04ae-0209-4053-8c4a-fae42f11b7ba reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:13:18.946376Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14123" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 to ip-10-0-132-145.ec2.internal metadata: creationTimestamp: "2026-07-01T18:13:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:13:18Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d061cbad237 namespace: kserve-ci-e2e-test resourceVersion: "14128" uid: 45926073-22b8-4749-ae51-21eab201a630 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-849956d989-wqhnw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:19Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:13:19Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-raw-sklearn-batcher-a980c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:13:19Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d0624a99ea0 namespace: kserve-ci-e2e-test resourceVersion: "14139" uid: 9e2975bc-25c2-4a17-a5b4-7e9361baba6b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:19Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14135" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:13:19Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:13:19Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d065a86a0bc namespace: kserve-ci-e2e-test resourceVersion: "14150" uid: 3118fdb0-720b-402f-b512-46abf49d08ae reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:13:20Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:13:20Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d065c14a546 namespace: kserve-ci-e2e-test resourceVersion: "14152" uid: 840f8ef6-a080-4572-93e7-9f6ffda98339 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:13:20Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:13:20Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d066032c932 namespace: kserve-ci-e2e-test resourceVersion: "14153" uid: 19054ff7-f0bb-4e57-ba8f-9d502fe22100 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:13:20Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:13:20Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d0660a96172 namespace: kserve-ci-e2e-test resourceVersion: "14154" uid: 0500b51c-c56f-41e6-9eb4-23dee03e6857 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:13:23Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" metadata: creationTimestamp: "2026-07-01T18:13:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:13:23Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d07469bf536 namespace: kserve-ci-e2e-test resourceVersion: "14184" uid: 8c6b5a15-a4e9-41e6-8049-a96fbd5c6d80 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:13:38Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" in 14.982s (14.982s including waiting). Image size: 1559186858 bytes.' metadata: creationTimestamp: "2026-07-01T18:13:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:13:38Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d0ac3a1f815 namespace: kserve-ci-e2e-test resourceVersion: "14299" uid: 98242184-0d90-4a18-bead-fc4c222adabf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:13:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:13:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:13:38Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d0ac73692a4 namespace: kserve-ci-e2e-test resourceVersion: "14300" uid: 9f770d2f-4574-40f1-a1d0-10f510f90860 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:13:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:13:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:13:38Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d0ac7b0bb5d namespace: kserve-ci-e2e-test resourceVersion: "14302" uid: b2d658d2-90c4-4d94-81a1-2575b667f8ad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:13:38Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T18:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:13:39Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d0ac7bd4b99 namespace: kserve-ci-e2e-test resourceVersion: "14303" uid: 9f386c27-3670-481b-8439-0d44b5605468 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:13:41Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.783s (2.783s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T18:13:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:13:41Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d0b6dab8471 namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 6ab05e23-8ea3-453e-9c1a-2efc73d73610 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:13:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:13:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:13:41Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d0b7199a6ad namespace: kserve-ci-e2e-test resourceVersion: "14320" uid: 00b4d8bf-0699-406d-bac9-f7db524c7dc2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:13:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:13:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:13:41Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d0b72225089 namespace: kserve-ci-e2e-test resourceVersion: "14321" uid: a2f8bf70-ca04-4441-93c8-9916430c6020 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:13:41Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:2fd41eb0f45027343da0a3e7ab10342a4e3f1970b88cf54fcc671e20be2ec985" metadata: creationTimestamp: "2026-07-01T18:13:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:13:41Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d0b722eb8a8 namespace: kserve-ci-e2e-test resourceVersion: "14322" uid: f5e2777b-d312-4a95-b2ce-63f973640dca reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:13:44Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:2fd41eb0f45027343da0a3e7ab10342a4e3f1970b88cf54fcc671e20be2ec985" in 3.111s (3.111s including waiting). Image size: 239994467 bytes.' metadata: creationTimestamp: "2026-07-01T18:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:13:44Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d0c2b9e203e namespace: kserve-ci-e2e-test resourceVersion: "14341" uid: 2e0f14d1-ae90-4cc6-b662-3250c31e960a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:13:45Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-01T18:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:13:45Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d0c2ee0bb45 namespace: kserve-ci-e2e-test resourceVersion: "14342" uid: 8dce4d17-f91d-4e41-bd4f-7960f5826c4c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:13:45Z" message: Started container agent metadata: creationTimestamp: "2026-07-01T18:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:13:45Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d0c2f5ef16b namespace: kserve-ci-e2e-test resourceVersion: "14343" uid: ca06a17a-98fd-4d35-86f9-92cd89d0ea18 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T18:13:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:15:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:08Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d0c314f7d4f namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: 5587b743-a2da-42e1-8837-da563cfd9f7e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T18:13:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:15:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:08Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d0c6d2dbe47 namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: d4915f3c-665e-41e9-9d0b-a325f26fa0c6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:15:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:04Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d1e98d14616 namespace: kserve-ci-e2e-test resourceVersion: "14972" uid: b0871e3f-3618-40d4-a938-7d3cc3a5dbc4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:15:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:04Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d1e98d23bfe namespace: kserve-ci-e2e-test resourceVersion: "14979" uid: 919e9f81-ef3c-4b73-be2c-5fc94be2c357 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:15:04Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:04Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d1e98d281bf namespace: kserve-ci-e2e-test resourceVersion: "14983" uid: fa568ebb-3307-4f43-b068-4daf2f03ce16 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:15:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 8e027644-5b9a-48ae-a225-f800e0d29195 kind: Event lastTimestamp: "2026-07-01T18:15:16Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:15:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:16Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5.18be3d1f0d8f7250 namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: f490b744-a9ef-4f24-b29e-e72a7cb11cf2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5 namespace: kserve-ci-e2e-test resourceVersion: "14116" uid: ad2ee5af-0d1d-433d-be64-1abb8380811d kind: Event lastTimestamp: "2026-07-01T18:13:18Z" message: 'Created pod: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5-xxph5' metadata: creationTimestamp: "2026-07-01T18:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:13:18Z" name: isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5.18be3d061c421395 namespace: kserve-ci-e2e-test resourceVersion: "14124" uid: 01bcb2e3-5531-4c61-b522-12ca349e90a7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-a980c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14115" uid: a2f15d3c-f499-4fea-a9cf-09f09e979312 kind: Event lastTimestamp: "2026-07-01T18:13:18Z" message: Scaled up replica set isvc-raw-sklearn-batcher-a980c-predictor-7db9b49dc5 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:13:18Z" name: isvc-raw-sklearn-batcher-a980c-predictor.18be3d0616fdf960 namespace: kserve-ci-e2e-test resourceVersion: "14117" uid: 9eb4efc7-7ba4-4411-a11a-b09f53f2bc41 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:13:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-a980c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14127" uid: bbc30464-6d61-4a59-a8ed-9bec16526410 kind: Event lastTimestamp: "2026-07-01T18:14:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:13:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:14:19Z" name: isvc-raw-sklearn-batcher-a980c-predictor.18be3d099c3f462c namespace: kserve-ci-e2e-test resourceVersion: "14595" uid: 7edc8f02-ac75-46ce-8c5b-a0ac11fae704 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:13:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-a980c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14127" uid: bbc30464-6d61-4a59-a8ed-9bec16526410 kind: Event lastTimestamp: "2026-07-01T18:14:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:13:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:14:19Z" name: isvc-raw-sklearn-batcher-a980c-predictor.18be3d099ca2e2a7 namespace: kserve-ci-e2e-test resourceVersion: "14596" uid: 74b41f85-a62f-4372-a49e-b0f92eceeae2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:14:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-a980c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: bbc30464-6d61-4a59-a8ed-9bec16526410 kind: Event lastTimestamp: "2026-07-01T18:15:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:15:04Z" name: isvc-raw-sklearn-batcher-a980c-predictor.18be3d179866f6b0 namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: f34b596d-47a9-43bb-8e39-22ba4fa48252 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:14:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-a980c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: bbc30464-6d61-4a59-a8ed-9bec16526410 kind: Event lastTimestamp: "2026-07-01T18:15:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:15:04Z" name: isvc-raw-sklearn-batcher-a980c-predictor.18be3d1798ca264c namespace: kserve-ci-e2e-test resourceVersion: "14956" uid: 05d92cc1-d784-40c0-87f8-e9b5dd4c750b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T18:13:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-a980c namespace: kserve-ci-e2e-test resourceVersion: "14105" uid: 8d4e8d25-d746-442f-a935-b75b37ef7e44 kind: Event lastTimestamp: "2026-07-01T18:14:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:14:58Z" name: isvc-raw-sklearn-batcher-a980c.18be3d061d401335 namespace: kserve-ci-e2e-test resourceVersion: "14892" uid: fb355519-2fe4-4531-ae34-5d4f480d8de0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-a980c namespace: kserve-ci-e2e-test resourceVersion: "14105" uid: 8d4e8d25-d746-442f-a935-b75b37ef7e44 kind: Event lastTimestamp: "2026-07-01T18:13:18Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-a980c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-a980c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:13:18Z" name: isvc-raw-sklearn-batcher-a980c.18be3d061dacc18e namespace: kserve-ci-e2e-test resourceVersion: "14133" uid: 37a2d0a3-2f1b-499b-9294-07df1c2d91be reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:13:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-a980c namespace: kserve-ci-e2e-test resourceVersion: "14105" uid: 8d4e8d25-d746-442f-a935-b75b37ef7e44 kind: Event lastTimestamp: "2026-07-01T18:13:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-a980c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:13:18Z" name: isvc-raw-sklearn-batcher-a980c.18be3d061dad03c3 namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: fdfcdcba-3df5-4a45-ad86-ca8475fa9cdf reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:14:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-a980c namespace: kserve-ci-e2e-test resourceVersion: "14890" uid: 8d4e8d25-d746-442f-a935-b75b37ef7e44 kind: Event lastTimestamp: "2026-07-01T18:14:58Z" message: InferenceService [isvc-raw-sklearn-batcher-a980c] is Ready metadata: creationTimestamp: "2026-07-01T18:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:14:58Z" name: isvc-raw-sklearn-batcher-a980c.18be3d1d387960b6 namespace: kserve-ci-e2e-test resourceVersion: "14891" uid: 2cc930e1-eabb-4ace-9777-d42adb24a182 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:25:05.592846Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-72dc06-predictor-6477d9654f-jlnkh namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: 817dac3b-5c8a-4d5d-b0f3-febe0ade7c95 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-72dc06-predictor-6477d9654f-jlnkh to ip-10-0-132-145.ec2.internal metadata: creationTimestamp: "2026-07-01T18:25:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:25:05Z" name: isvc-secondary-72dc06-predictor-6477d9654f-jlnkh.18be3daaa42403ff namespace: kserve-ci-e2e-test resourceVersion: "20123" uid: 62f9c031-228d-468d-a911-9787149730a7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-849956d989-wqhnw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-72dc06-predictor-6477d9654f-jlnkh namespace: kserve-ci-e2e-test resourceVersion: "20127" uid: 817dac3b-5c8a-4d5d-b0f3-febe0ade7c95 kind: Event lastTimestamp: "2026-07-01T18:25:06Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:25:06Z" name: isvc-secondary-72dc06-predictor-6477d9654f-jlnkh.18be3daabe0639b5 namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: b343fe9c-d87c-4fcd-b363-b18dac3f1368 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:25:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-72dc06-predictor-6477d9654f-jlnkh namespace: kserve-ci-e2e-test resourceVersion: "20120" uid: 817dac3b-5c8a-4d5d-b0f3-febe0ade7c95 kind: Event lastTimestamp: "2026-07-01T18:25:13Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:13Z" name: isvc-secondary-72dc06-predictor-6477d9654f-jlnkh.18be3daabf636e25 namespace: kserve-ci-e2e-test resourceVersion: "20200" uid: b9018cf8-92f1-4f4e-91fe-e6c5c723e7a0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:25:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-72dc06-predictor-6477d9654f-jlnkh namespace: kserve-ci-e2e-test resourceVersion: "20120" uid: 817dac3b-5c8a-4d5d-b0f3-febe0ade7c95 kind: Event lastTimestamp: "2026-07-01T18:25:13Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:13Z" name: isvc-secondary-72dc06-predictor-6477d9654f-jlnkh.18be3daac23425b6 namespace: kserve-ci-e2e-test resourceVersion: "20204" uid: 14263fd0-b210-4434-b8f3-94a5979830e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:25:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-72dc06-predictor-6477d9654f-jlnkh namespace: kserve-ci-e2e-test resourceVersion: "20120" uid: 817dac3b-5c8a-4d5d-b0f3-febe0ade7c95 kind: Event lastTimestamp: "2026-07-01T18:25:13Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:13Z" name: isvc-secondary-72dc06-predictor-6477d9654f-jlnkh.18be3daac2aee41a namespace: kserve-ci-e2e-test resourceVersion: "20205" uid: f1ee2dd5-fab3-417c-a7cd-c25280200924 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-72dc06-predictor-6477d9654f-jlnkh namespace: kserve-ci-e2e-test resourceVersion: "20120" uid: 817dac3b-5c8a-4d5d-b0f3-febe0ade7c95 kind: Event lastTimestamp: "2026-07-01T18:25:19Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-72dc06-predictor-6477d9654f-jlnkh_kserve-ci-e2e-test(817dac3b-5c8a-4d5d-b0f3-febe0ade7c95) metadata: creationTimestamp: "2026-07-01T18:25:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:19Z" name: isvc-secondary-72dc06-predictor-6477d9654f-jlnkh.18be3dadd5981a6a namespace: kserve-ci-e2e-test resourceVersion: "20249" uid: 3247a4f8-bc57-4b25-8726-83d42525eff7 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-72dc06-predictor-6477d9654f namespace: kserve-ci-e2e-test resourceVersion: "20109" uid: 7d2d7a0e-14db-43db-8595-0504efdbb10e kind: Event lastTimestamp: "2026-07-01T18:25:05Z" message: 'Created pod: isvc-secondary-72dc06-predictor-6477d9654f-jlnkh' metadata: creationTimestamp: "2026-07-01T18:25:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:25:05Z" name: isvc-secondary-72dc06-predictor-6477d9654f.18be3daaa3b20d45 namespace: kserve-ci-e2e-test resourceVersion: "20119" uid: 38da0f4b-9b84-4c1b-87b3-7fabce285a2b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-72dc06-predictor namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 15714159-8a4f-4061-95bc-a7903bce2e44 kind: Event lastTimestamp: "2026-07-01T18:25:05Z" message: Scaled up replica set isvc-secondary-72dc06-predictor-6477d9654f from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:25:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:25:05Z" name: isvc-secondary-72dc06-predictor.18be3daa9f1ea46f namespace: kserve-ci-e2e-test resourceVersion: "20110" uid: bca9e462-9cd3-4bcf-948e-03330c5814e2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-72dc06-predictor namespace: kserve-ci-e2e-test resourceVersion: "20108" uid: 16ac92cd-8d76-4aed-b2f4-209f81601980 kind: Event lastTimestamp: "2026-07-01T18:25:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:25:20Z" name: isvc-secondary-72dc06-predictor.18be3dae1d8f0b40 namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: b4e4cd31-4816-4fa8-91d9-fbada7f834ff reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-72dc06-predictor namespace: kserve-ci-e2e-test resourceVersion: "20108" uid: 16ac92cd-8d76-4aed-b2f4-209f81601980 kind: Event lastTimestamp: "2026-07-01T18:25:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:25:20Z" name: isvc-secondary-72dc06-predictor.18be3dae1deb385b namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 19566c35-78bf-4d10-930b-0e69a7a68ea6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T18:25:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-72dc06 namespace: kserve-ci-e2e-test resourceVersion: "20099" uid: 6f39ac88-8693-4c14-93a8-384c93bd873e kind: Event lastTimestamp: "2026-07-01T18:25:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:25:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:25:19Z" name: isvc-secondary-72dc06.18be3daa9f252245 namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: bf284c79-be2e-4542-9c26-59444d43dfba reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-72dc06 namespace: kserve-ci-e2e-test resourceVersion: "20099" uid: 6f39ac88-8693-4c14-93a8-384c93bd873e kind: Event lastTimestamp: "2026-07-01T18:25:05Z" message: 'Failed to update status for InferenceService "isvc-secondary-72dc06": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-72dc06": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:25:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:25:05Z" name: isvc-secondary-72dc06.18be3daa9f9b2b39 namespace: kserve-ci-e2e-test resourceVersion: "20113" uid: cc11b49c-16ae-48e5-8f1e-8d1720f3131d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-72dc06 namespace: kserve-ci-e2e-test resourceVersion: "20099" uid: 6f39ac88-8693-4c14-93a8-384c93bd873e kind: Event lastTimestamp: "2026-07-01T18:25:05Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-72dc06": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:25:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:25:05Z" name: isvc-secondary-72dc06.18be3daa9f9b7974 namespace: kserve-ci-e2e-test resourceVersion: "20114" uid: 50c63018-b31a-41d0-bfed-cbe169bd3da4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:15:04.115101Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf namespace: kserve-ci-e2e-test resourceVersion: "14969" uid: 9994897a-b72e-4cdc-b9f3-a77e709543c5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf to ip-10-0-132-145.ec2.internal metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:15:04Z" name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf.18be3d1e9945fe5e namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 6d5c91a7-61e6-4ef9-ba9e-5affe3625780 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-849956d989-wqhnw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: 9994897a-b72e-4cdc-b9f3-a77e709543c5 kind: Event lastTimestamp: "2026-07-01T18:15:04Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:15:04Z" name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf.18be3d1eb351ced8 namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: f0b25750-64f0-4fd3-b2e7-4ec3f3251f39 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: 9994897a-b72e-4cdc-b9f3-a77e709543c5 kind: Event lastTimestamp: "2026-07-01T18:15:04Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:04Z" name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf.18be3d1eb4cb74df namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: de9df557-ddbd-4ed4-885f-0ba9b8058252 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: 9994897a-b72e-4cdc-b9f3-a77e709543c5 kind: Event lastTimestamp: "2026-07-01T18:15:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:04Z" name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf.18be3d1eb93a396f namespace: kserve-ci-e2e-test resourceVersion: "15027" uid: 7cfc5585-08d2-462c-b07a-2c59d9d33ec0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: 9994897a-b72e-4cdc-b9f3-a77e709543c5 kind: Event lastTimestamp: "2026-07-01T18:15:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:04Z" name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf.18be3d1eb9a8794b namespace: kserve-ci-e2e-test resourceVersion: "15029" uid: f31cbbba-e1e1-44da-a087-ae95c8be79bc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: 9994897a-b72e-4cdc-b9f3-a77e709543c5 kind: Event lastTimestamp: "2026-07-01T18:15:08Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:15:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:08Z" name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf.18be3d1f99f7edfd namespace: kserve-ci-e2e-test resourceVersion: "15071" uid: 92ae4635-e93b-47d8-9778-3d04eaa27d30 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: 9994897a-b72e-4cdc-b9f3-a77e709543c5 kind: Event lastTimestamp: "2026-07-01T18:15:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:15:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:08Z" name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf.18be3d1f9f9d15bc namespace: kserve-ci-e2e-test resourceVersion: "15077" uid: 1a0dc7a6-ac73-4949-ad0e-f83dc046eee0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: 9994897a-b72e-4cdc-b9f3-a77e709543c5 kind: Event lastTimestamp: "2026-07-01T18:15:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:15:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:08Z" name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf.18be3d1fa00b1e5a namespace: kserve-ci-e2e-test resourceVersion: "15078" uid: fbd0d9c5-9bc7-45bd-a0e8-d39cfa67ee95 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: 9994897a-b72e-4cdc-b9f3-a77e709543c5 kind: Event lastTimestamp: "2026-07-01T18:15:08Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:15:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:08Z" name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf.18be3d1fa014e0c8 namespace: kserve-ci-e2e-test resourceVersion: "15079" uid: c04e8ee0-c6b8-48dd-a2f0-6e58385fef1d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: 9994897a-b72e-4cdc-b9f3-a77e709543c5 kind: Event lastTimestamp: "2026-07-01T18:15:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:15:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:08Z" name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf.18be3d1fa38e127f namespace: kserve-ci-e2e-test resourceVersion: "15083" uid: 7b48a5ea-0386-4a9e-98b4-4f0f99010f84 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: 9994897a-b72e-4cdc-b9f3-a77e709543c5 kind: Event lastTimestamp: "2026-07-01T18:15:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:15:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:08Z" name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf.18be3d1fa42c7e5c namespace: kserve-ci-e2e-test resourceVersion: "15084" uid: 045abcb5-a9fd-4305-814f-bd6c24a1a08a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T18:15:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: 9994897a-b72e-4cdc-b9f3-a77e709543c5 kind: Event lastTimestamp: "2026-07-01T18:16:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:15:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:57Z" name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf.18be3d201253d1bc namespace: kserve-ci-e2e-test resourceVersion: "16104" uid: 2d39208c-0aec-4e06-9233-e0f3dc830b76 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: 9994897a-b72e-4cdc-b9f3-a77e709543c5 kind: Event lastTimestamp: "2026-07-01T18:16:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:54Z" name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf.18be3d38448c755f namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: 0bec6b17-6209-4c59-9d1a-3400f13e8086 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: 9994897a-b72e-4cdc-b9f3-a77e709543c5 kind: Event lastTimestamp: "2026-07-01T18:16:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:54Z" name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf.18be3d38448d1f23 namespace: kserve-ci-e2e-test resourceVersion: "15986" uid: 101fff7d-0ef4-444d-ae1c-fffad9afde4e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: 9994897a-b72e-4cdc-b9f3-a77e709543c5 kind: Event lastTimestamp: "2026-07-01T18:16:56Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:56Z" name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf.18be3d38c0a2e2ac namespace: kserve-ci-e2e-test resourceVersion: "16091" uid: ba7932b6-1d30-4d3f-9e29-861820aa6dec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495 namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: 56085d13-972d-49c8-b0bd-356342ad03c8 kind: Event lastTimestamp: "2026-07-01T18:15:04Z" message: 'Created pod: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495-z84rf' metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:15:04Z" name: isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495.18be3d1e98d46231 namespace: kserve-ci-e2e-test resourceVersion: "14973" uid: 32bb6681-1f29-4463-89d2-9a87627501bc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-ebf1b-predictor namespace: kserve-ci-e2e-test resourceVersion: "14939" uid: 819c6366-e9ea-40f2-a906-57c9fce6b2fa kind: Event lastTimestamp: "2026-07-01T18:15:04Z" message: Scaled up replica set isvc-sklearn-graph-raw-ebf1b-predictor-85b9f95495 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:15:04Z" name: isvc-sklearn-graph-raw-ebf1b-predictor.18be3d1e94267732 namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: 47eabf36-691a-479d-b0c7-b485179863f3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:15:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-ebf1b-predictor namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: 49df54c7-e067-4b0b-87fe-04b893fab8b7 kind: Event lastTimestamp: "2026-07-01T18:15:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:15:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:15:49Z" name: isvc-sklearn-graph-raw-ebf1b-predictor.18be3d2212a7059a namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: 5f091387-9d72-4682-aa8e-3187bd26a1a1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:15:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-ebf1b-predictor namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: 49df54c7-e067-4b0b-87fe-04b893fab8b7 kind: Event lastTimestamp: "2026-07-01T18:15:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:15:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:15:49Z" name: isvc-sklearn-graph-raw-ebf1b-predictor.18be3d2213131333 namespace: kserve-ci-e2e-test resourceVersion: "15407" uid: 15f75142-8d46-495b-81ed-b708b7c1cb0f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:15:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-ebf1b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15172" uid: 49df54c7-e067-4b0b-87fe-04b893fab8b7 kind: Event lastTimestamp: "2026-07-01T18:16:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:15:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:16:49Z" name: isvc-sklearn-graph-raw-ebf1b-predictor.18be3d259208ca69 namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: 88df6a4f-c7ca-4eaa-8bd4-ebfc64865a1b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:15:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-ebf1b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15172" uid: 49df54c7-e067-4b0b-87fe-04b893fab8b7 kind: Event lastTimestamp: "2026-07-01T18:16:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:15:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:16:49Z" name: isvc-sklearn-graph-raw-ebf1b-predictor.18be3d259272937b namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: 11cac506-b87e-48f3-9876-a6db58c47d92 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T18:15:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-ebf1b namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: cc5ba98f-71f4-4f9a-96bd-06729394d750 kind: Event lastTimestamp: "2026-07-01T18:16:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:16:28Z" name: isvc-sklearn-graph-raw-ebf1b.18be3d1e9428b8cd namespace: kserve-ci-e2e-test resourceVersion: "15689" uid: 8c98fe35-8070-481a-a671-654b48de1884 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-ebf1b namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: cc5ba98f-71f4-4f9a-96bd-06729394d750 kind: Event lastTimestamp: "2026-07-01T18:15:04Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-ebf1b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-ebf1b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:15:04Z" name: isvc-sklearn-graph-raw-ebf1b.18be3d1e94b903ac namespace: kserve-ci-e2e-test resourceVersion: "14945" uid: c2789c53-07b3-42ea-91ba-08b0aa20e454 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-ebf1b namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: cc5ba98f-71f4-4f9a-96bd-06729394d750 kind: Event lastTimestamp: "2026-07-01T18:16:27Z" message: InferenceService [isvc-sklearn-graph-raw-ebf1b] is Ready metadata: creationTimestamp: "2026-07-01T18:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:16:27Z" name: isvc-sklearn-graph-raw-ebf1b.18be3d321f62f699 namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: df963fbb-c374-44bd-8ea9-a6142e71b35d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:16:54.435565Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj namespace: kserve-ci-e2e-test resourceVersion: "15995" uid: ca6c4380-bcbd-4dd3-aca3-7812e1e4d829 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj to ip-10-0-132-145.ec2.internal metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:16:54Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj.18be3d3848e2a041 namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: 083190ef-a49f-4ef9-8735-dcdabca18579 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-849956d989-wqhnw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:54Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: ca6c4380-bcbd-4dd3-aca3-7812e1e4d829 kind: Event lastTimestamp: "2026-07-01T18:16:54Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:16:54Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj.18be3d38634c20cd namespace: kserve-ci-e2e-test resourceVersion: "16059" uid: 794af983-e1d2-466b-901c-0266c0f0833d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: ca6c4380-bcbd-4dd3-aca3-7812e1e4d829 kind: Event lastTimestamp: "2026-07-01T18:16:54Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:54Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj.18be3d3864d409ac namespace: kserve-ci-e2e-test resourceVersion: "16061" uid: 03a74748-48fa-4870-a761-5f281110b6c1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: ca6c4380-bcbd-4dd3-aca3-7812e1e4d829 kind: Event lastTimestamp: "2026-07-01T18:16:54Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:54Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj.18be3d3868fc6df2 namespace: kserve-ci-e2e-test resourceVersion: "16065" uid: 03f9fe96-6434-40ae-90bf-1335f649cda0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: ca6c4380-bcbd-4dd3-aca3-7812e1e4d829 kind: Event lastTimestamp: "2026-07-01T18:16:54Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:54Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj.18be3d38696d23dd namespace: kserve-ci-e2e-test resourceVersion: "16066" uid: 905a1b27-f458-4c4c-9b0b-87970a58d18d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: ca6c4380-bcbd-4dd3-aca3-7812e1e4d829 kind: Event lastTimestamp: "2026-07-01T18:16:58Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:16:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:58Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj.18be3d3957ec99c4 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: 5b5d198d-e545-4e83-8b3c-bd1e02429bf7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: ca6c4380-bcbd-4dd3-aca3-7812e1e4d829 kind: Event lastTimestamp: "2026-07-01T18:16:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:16:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:59Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj.18be3d395b295d2e namespace: kserve-ci-e2e-test resourceVersion: "16126" uid: 07d9dc61-09a3-420c-8607-936b3dbb1cc8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: ca6c4380-bcbd-4dd3-aca3-7812e1e4d829 kind: Event lastTimestamp: "2026-07-01T18:16:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:16:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:59Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj.18be3d395b9f5435 namespace: kserve-ci-e2e-test resourceVersion: "16130" uid: dfd49d12-91f8-4884-a5c1-f2f88b7f3f58 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: ca6c4380-bcbd-4dd3-aca3-7812e1e4d829 kind: Event lastTimestamp: "2026-07-01T18:16:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:16:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:59Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj.18be3d395ba9c49d namespace: kserve-ci-e2e-test resourceVersion: "16132" uid: e36f7357-e94f-4d11-af1c-4c183f28f44b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: ca6c4380-bcbd-4dd3-aca3-7812e1e4d829 kind: Event lastTimestamp: "2026-07-01T18:16:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:16:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:59Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj.18be3d395ef97c60 namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: e79eea6a-9e54-43d4-8b4c-64ed6dce27cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: ca6c4380-bcbd-4dd3-aca3-7812e1e4d829 kind: Event lastTimestamp: "2026-07-01T18:16:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:16:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:59Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj.18be3d395f640efa namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: e650a031-f9d8-4889-8fa8-a51c5292adbd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T18:16:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: ca6c4380-bcbd-4dd3-aca3-7812e1e4d829 kind: Event lastTimestamp: "2026-07-01T18:18:47Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:17:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:47Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj.18be3d3994890e70 namespace: kserve-ci-e2e-test resourceVersion: "17104" uid: 573ce240-c2bf-4916-8859-b41383b91410 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: ca6c4380-bcbd-4dd3-aca3-7812e1e4d829 kind: Event lastTimestamp: "2026-07-01T18:18:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:18:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:44Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj.18be3d51f99e021d namespace: kserve-ci-e2e-test resourceVersion: "17052" uid: 94568403-fd60-4a0e-9fea-ef71c8b112d8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: ca6c4380-bcbd-4dd3-aca3-7812e1e4d829 kind: Event lastTimestamp: "2026-07-01T18:18:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:18:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:44Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj.18be3d51f99e7829 namespace: kserve-ci-e2e-test resourceVersion: "17059" uid: 1f5feabe-da0a-4408-8295-a2adc3f6e521 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: ca6c4380-bcbd-4dd3-aca3-7812e1e4d829 kind: Event lastTimestamp: "2026-07-01T18:18:47Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:47Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj.18be3d527f2c112c namespace: kserve-ci-e2e-test resourceVersion: "17103" uid: c65f159f-ff83-4075-98ba-80c3e2c13d72 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 1dc9e39a-385b-4460-9c79-a9debf5507e5 kind: Event lastTimestamp: "2026-07-01T18:16:54Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f-z9hdj' metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:16:54Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f.18be3d38486025ad namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: 8fc58640-5cdb-4fbe-9903-8880fe11a6dc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-c9b93-predictor namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: 56e3b030-362a-4c70-a9b2-081ee59e491c kind: Event lastTimestamp: "2026-07-01T18:16:54Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-c9b93-predictor-7f4844db9f from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:16:54Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor.18be3d3843dcc5e2 namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: 91d27c60-4612-4140-aeaa-419c36c1bff3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:17:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-c9b93-predictor namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: e5f5fbbf-c52d-45aa-a1e1-4f3b9102cbf5 kind: Event lastTimestamp: "2026-07-01T18:17:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:17:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:17:39Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor.18be3d3bc2c24c5a namespace: kserve-ci-e2e-test resourceVersion: "16450" uid: 82573f4e-4046-41e2-a8ef-91dbf5c1b8e2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:17:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-c9b93-predictor namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: e5f5fbbf-c52d-45aa-a1e1-4f3b9102cbf5 kind: Event lastTimestamp: "2026-07-01T18:17:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:17:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:17:39Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor.18be3d3bc32a74a8 namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: cc805bf4-4706-4a2f-8121-41ddb37a877f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:17:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-c9b93-predictor namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: e5f5fbbf-c52d-45aa-a1e1-4f3b9102cbf5 kind: Event lastTimestamp: "2026-07-01T18:18:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:18:39Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor.18be3d463fbbe183 namespace: kserve-ci-e2e-test resourceVersion: "16938" uid: 4c90004f-1999-4fb9-96d7-881bb08a04f2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:17:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-c9b93-predictor namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: e5f5fbbf-c52d-45aa-a1e1-4f3b9102cbf5 kind: Event lastTimestamp: "2026-07-01T18:18:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:18:39Z" name: isvc-sklearn-graph-raw-hpa-c9b93-predictor.18be3d46401eaff8 namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: e4b8054b-65fd-4543-95b6-f35caf0b6729 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T18:16:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-c9b93 namespace: kserve-ci-e2e-test resourceVersion: "15953" uid: fba0c8f2-18fa-4e50-836b-457020b02016 kind: Event lastTimestamp: "2026-07-01T18:18:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:18:17Z" name: isvc-sklearn-graph-raw-hpa-c9b93.18be3d3843e7d8a5 namespace: kserve-ci-e2e-test resourceVersion: "16753" uid: 51886372-99b6-43ed-92f0-16ff56133998 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-c9b93 namespace: kserve-ci-e2e-test resourceVersion: "15953" uid: fba0c8f2-18fa-4e50-836b-457020b02016 kind: Event lastTimestamp: "2026-07-01T18:16:54Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-c9b93": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-c9b93": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:16:54Z" name: isvc-sklearn-graph-raw-hpa-c9b93.18be3d3844559236 namespace: kserve-ci-e2e-test resourceVersion: "15981" uid: 258e55e3-bf22-432c-aaaf-c30c241e1764 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-c9b93 namespace: kserve-ci-e2e-test resourceVersion: "16751" uid: fba0c8f2-18fa-4e50-836b-457020b02016 kind: Event lastTimestamp: "2026-07-01T18:18:17Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-c9b93] is Ready metadata: creationTimestamp: "2026-07-01T18:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:18:17Z" name: isvc-sklearn-graph-raw-hpa-c9b93.18be3d4b8b602bd4 namespace: kserve-ci-e2e-test resourceVersion: "16752" uid: be5b923c-b359-4d79-b965-ecd03bc9c074 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:20:40.073951Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj namespace: kserve-ci-e2e-test resourceVersion: "18053" uid: 781a7847-d3da-4c6c-a614-6e87c6179fc6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj to ip-10-0-132-145.ec2.internal metadata: creationTimestamp: "2026-07-01T18:20:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:20:40Z" name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj.18be3d6cd1fb2ca0 namespace: kserve-ci-e2e-test resourceVersion: "18060" uid: 79bae844-9671-478c-b517-d9aeeab60041 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-849956d989-wqhnw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj namespace: kserve-ci-e2e-test resourceVersion: "18076" uid: 781a7847-d3da-4c6c-a614-6e87c6179fc6 kind: Event lastTimestamp: "2026-07-01T18:20:40Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:20:40Z" name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj.18be3d6cec53606e namespace: kserve-ci-e2e-test resourceVersion: "18103" uid: 17e41d89-7a2c-40e9-9ce9-255710edb108 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj namespace: kserve-ci-e2e-test resourceVersion: "18056" uid: 781a7847-d3da-4c6c-a614-6e87c6179fc6 kind: Event lastTimestamp: "2026-07-01T18:20:40Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:40Z" name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj.18be3d6cedb7f557 namespace: kserve-ci-e2e-test resourceVersion: "18105" uid: facf6880-c1ee-4595-be38-7d8dd0cfa350 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj namespace: kserve-ci-e2e-test resourceVersion: "18056" uid: 781a7847-d3da-4c6c-a614-6e87c6179fc6 kind: Event lastTimestamp: "2026-07-01T18:20:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:40Z" name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj.18be3d6cf0527263 namespace: kserve-ci-e2e-test resourceVersion: "18106" uid: 3e5b4876-a04f-4ea7-bafd-d267ee75d98e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj namespace: kserve-ci-e2e-test resourceVersion: "18056" uid: 781a7847-d3da-4c6c-a614-6e87c6179fc6 kind: Event lastTimestamp: "2026-07-01T18:20:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:40Z" name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj.18be3d6cf0d10979 namespace: kserve-ci-e2e-test resourceVersion: "18107" uid: ff0c760a-6e86-4a2d-8c03-994a24eaf5c4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj namespace: kserve-ci-e2e-test resourceVersion: "18056" uid: 781a7847-d3da-4c6c-a614-6e87c6179fc6 kind: Event lastTimestamp: "2026-07-01T18:20:45Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:20:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:45Z" name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj.18be3d6dfc1b162a namespace: kserve-ci-e2e-test resourceVersion: "18142" uid: 7e3fda41-dcea-42b8-8f60-ad29c2857194 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj namespace: kserve-ci-e2e-test resourceVersion: "18056" uid: 781a7847-d3da-4c6c-a614-6e87c6179fc6 kind: Event lastTimestamp: "2026-07-01T18:20:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:20:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:45Z" name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj.18be3d6dffa3e84d namespace: kserve-ci-e2e-test resourceVersion: "18145" uid: 6889d03d-a64f-4393-bee3-86f1200cc64c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj namespace: kserve-ci-e2e-test resourceVersion: "18056" uid: 781a7847-d3da-4c6c-a614-6e87c6179fc6 kind: Event lastTimestamp: "2026-07-01T18:20:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:20:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:45Z" name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj.18be3d6e0022ab77 namespace: kserve-ci-e2e-test resourceVersion: "18148" uid: c4c91ee3-1b6b-455a-b18f-b7f13b09ef55 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj namespace: kserve-ci-e2e-test resourceVersion: "18056" uid: 781a7847-d3da-4c6c-a614-6e87c6179fc6 kind: Event lastTimestamp: "2026-07-01T18:20:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:20:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:45Z" name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj.18be3d6e002dfda4 namespace: kserve-ci-e2e-test resourceVersion: "18149" uid: 45bd9a62-4089-4977-bd56-3d8b2688dab4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj namespace: kserve-ci-e2e-test resourceVersion: "18056" uid: 781a7847-d3da-4c6c-a614-6e87c6179fc6 kind: Event lastTimestamp: "2026-07-01T18:20:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:20:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:45Z" name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj.18be3d6e0425a70a namespace: kserve-ci-e2e-test resourceVersion: "18152" uid: 4ef23bcb-6314-4c04-954b-c9f4f86ff192 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj namespace: kserve-ci-e2e-test resourceVersion: "18056" uid: 781a7847-d3da-4c6c-a614-6e87c6179fc6 kind: Event lastTimestamp: "2026-07-01T18:20:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:20:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:45Z" name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj.18be3d6e04a3320d namespace: kserve-ci-e2e-test resourceVersion: "18153" uid: 81017a59-b4dd-4a70-88d0-883f44170436 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 19 eventTime: null firstTimestamp: "2026-07-01T18:20:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj namespace: kserve-ci-e2e-test resourceVersion: "18056" uid: 781a7847-d3da-4c6c-a614-6e87c6179fc6 kind: Event lastTimestamp: "2026-07-01T18:23:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:20:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:32Z" name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj.18be3d6e38222fcb namespace: kserve-ci-e2e-test resourceVersion: "19377" uid: ed949614-d2c1-43ba-8f71-5cd20441b828 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj namespace: kserve-ci-e2e-test resourceVersion: "18056" uid: 781a7847-d3da-4c6c-a614-6e87c6179fc6 kind: Event lastTimestamp: "2026-07-01T18:23:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:23:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:30Z" name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj.18be3d9471604a28 namespace: kserve-ci-e2e-test resourceVersion: "19351" uid: 995eb2e1-7b92-43be-9373-ec9824392d0c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj namespace: kserve-ci-e2e-test resourceVersion: "18056" uid: 781a7847-d3da-4c6c-a614-6e87c6179fc6 kind: Event lastTimestamp: "2026-07-01T18:23:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:23:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:30Z" name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj.18be3d947160bc81 namespace: kserve-ci-e2e-test resourceVersion: "19352" uid: 93404983-8d21-44a4-bff8-ff531c43b449 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:23:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj namespace: kserve-ci-e2e-test resourceVersion: "18056" uid: 781a7847-d3da-4c6c-a614-6e87c6179fc6 kind: Event lastTimestamp: "2026-07-01T18:23:37Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:23:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:23:37Z" name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj.18be3d94deb4ed4b namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: cee1edf5-cb2d-41ec-8ad3-f54d50bfec7c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45 namespace: kserve-ci-e2e-test resourceVersion: "18034" uid: c91a9d5c-f39f-4f94-afe2-375b0407e2af kind: Event lastTimestamp: "2026-07-01T18:20:40Z" message: 'Created pod: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45-zkcvj' metadata: creationTimestamp: "2026-07-01T18:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:40Z" name: isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45.18be3d6cd16a87e3 namespace: kserve-ci-e2e-test resourceVersion: "18055" uid: 00eb6d88-0c17-408c-91dd-5da37a986974 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-7a9f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18032" uid: 5c6e9b93-747b-4228-883c-5ffc455d5193 kind: Event lastTimestamp: "2026-07-01T18:20:39Z" message: Scaled up replica set isvc-sklearn-scale-raw-7a9f1-predictor-79655cbf45 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:20:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:20:39Z" name: isvc-sklearn-scale-raw-7a9f1-predictor.18be3d6cccde737a namespace: kserve-ci-e2e-test resourceVersion: "18035" uid: 19ab207d-4dd2-48e6-8e60-fb51c8df79be reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:20:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-7a9f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18033" uid: c1f5d23e-028c-4d2d-9c01-86e5b6896c65 kind: Event lastTimestamp: "2026-07-01T18:21:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:25Z" name: isvc-sklearn-scale-raw-7a9f1-predictor.18be3d704b474a19 namespace: kserve-ci-e2e-test resourceVersion: "18451" uid: f778d814-2611-416c-b7f3-c190fc2099a7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:20:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-7a9f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18033" uid: c1f5d23e-028c-4d2d-9c01-86e5b6896c65 kind: Event lastTimestamp: "2026-07-01T18:21:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:20:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:21:25Z" name: isvc-sklearn-scale-raw-7a9f1-predictor.18be3d704baa32b1 namespace: kserve-ci-e2e-test resourceVersion: "18452" uid: 7f0697e5-c62f-4e6d-bdc0-3b1c43bb5a91 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T18:21:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-7a9f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18236" uid: c1f5d23e-028c-4d2d-9c01-86e5b6896c65 kind: Event lastTimestamp: "2026-07-01T18:23:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:23:25Z" name: isvc-sklearn-scale-raw-7a9f1-predictor.18be3d7ac90bbe5c namespace: kserve-ci-e2e-test resourceVersion: "19297" uid: c4ef98a5-4bec-4d75-8e2d-03a0bb742f44 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T18:21:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-7a9f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18236" uid: c1f5d23e-028c-4d2d-9c01-86e5b6896c65 kind: Event lastTimestamp: "2026-07-01T18:23:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:23:25Z" name: isvc-sklearn-scale-raw-7a9f1-predictor.18be3d7ac989c523 namespace: kserve-ci-e2e-test resourceVersion: "19298" uid: 622d4480-d313-44c7-9356-666b924166ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T18:20:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-7a9f1 namespace: kserve-ci-e2e-test resourceVersion: "18023" uid: 1f3c8293-6aa7-4281-9b6a-de8ceca496d3 kind: Event lastTimestamp: "2026-07-01T18:23:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:20:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:23:22Z" name: isvc-sklearn-scale-raw-7a9f1.18be3d6ccce2936b namespace: kserve-ci-e2e-test resourceVersion: "19283" uid: b66016c3-12fa-472f-8ce4-fe231ea58ade reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:20:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-7a9f1 namespace: kserve-ci-e2e-test resourceVersion: "18023" uid: 1f3c8293-6aa7-4281-9b6a-de8ceca496d3 kind: Event lastTimestamp: "2026-07-01T18:20:40Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-7a9f1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-7a9f1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:20:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:20:40Z" name: isvc-sklearn-scale-raw-7a9f1.18be3d6ccd3047e8 namespace: kserve-ci-e2e-test resourceVersion: "18093" uid: bea61e28-ea24-495d-a383-c685f80002e9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-7a9f1 namespace: kserve-ci-e2e-test resourceVersion: "18040" uid: 1f3c8293-6aa7-4281-9b6a-de8ceca496d3 kind: Event lastTimestamp: "2026-07-01T18:20:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-7a9f1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:20:40Z" name: isvc-sklearn-scale-raw-7a9f1.18be3d6cd690c7b8 namespace: kserve-ci-e2e-test resourceVersion: "18094" uid: 24eda9c7-51fa-4a54-adee-018461b876d3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:23:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-7a9f1 namespace: kserve-ci-e2e-test resourceVersion: "19281" uid: 1f3c8293-6aa7-4281-9b6a-de8ceca496d3 kind: Event lastTimestamp: "2026-07-01T18:23:22Z" message: InferenceService [isvc-sklearn-scale-raw-7a9f1] is Ready metadata: creationTimestamp: "2026-07-01T18:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:23:22Z" name: isvc-sklearn-scale-raw-7a9f1.18be3d92be91b11c namespace: kserve-ci-e2e-test resourceVersion: "19282" uid: 5eeb631d-c344-41a6-8d04-e8cf74960c61 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:15:04.193158Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: 337a6fc4-6f5e-4cb7-bf45-5fb46907b6b1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc to ip-10-0-132-145.ec2.internal metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:15:04Z" name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc.18be3d1e9ded0825 namespace: kserve-ci-e2e-test resourceVersion: "15006" uid: 2e67bed1-9b76-4fd5-80aa-01f875ff8f10 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-849956d989-wqhnw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc namespace: kserve-ci-e2e-test resourceVersion: "15013" uid: 337a6fc4-6f5e-4cb7-bf45-5fb46907b6b1 kind: Event lastTimestamp: "2026-07-01T18:15:04Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:15:04Z" name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc.18be3d1eb7eee78a namespace: kserve-ci-e2e-test resourceVersion: "15025" uid: a20d56d7-60cc-434e-bbbb-9487b5c8f338 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 337a6fc4-6f5e-4cb7-bf45-5fb46907b6b1 kind: Event lastTimestamp: "2026-07-01T18:15:04Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:04Z" name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc.18be3d1eb957db9e namespace: kserve-ci-e2e-test resourceVersion: "15028" uid: 60493fa7-9dca-4f72-8075-90d62584f07f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 337a6fc4-6f5e-4cb7-bf45-5fb46907b6b1 kind: Event lastTimestamp: "2026-07-01T18:15:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:04Z" name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc.18be3d1ebce61fc5 namespace: kserve-ci-e2e-test resourceVersion: "15032" uid: 80ac24a5-bc7e-4498-b1e5-411b0bfa525f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 337a6fc4-6f5e-4cb7-bf45-5fb46907b6b1 kind: Event lastTimestamp: "2026-07-01T18:15:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:04Z" name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc.18be3d1ebd4ef666 namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: fa9b4030-bed0-4c70-a3f6-16e10661afb0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 337a6fc4-6f5e-4cb7-bf45-5fb46907b6b1 kind: Event lastTimestamp: "2026-07-01T18:15:08Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-01T18:15:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:08Z" name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc.18be3d1f9a153926 namespace: kserve-ci-e2e-test resourceVersion: "15072" uid: e46ccf7e-72f3-4ad9-9b04-979b1e593070 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 337a6fc4-6f5e-4cb7-bf45-5fb46907b6b1 kind: Event lastTimestamp: "2026-07-01T18:15:28Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 20.568s (20.568s including waiting). Image size: 1305293052 bytes.' metadata: creationTimestamp: "2026-07-01T18:15:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:28Z" name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc.18be3d2464108ce0 namespace: kserve-ci-e2e-test resourceVersion: "15242" uid: 54afb625-a82c-4b89-8139-dcd9d82ab414 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 337a6fc4-6f5e-4cb7-bf45-5fb46907b6b1 kind: Event lastTimestamp: "2026-07-01T18:15:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:15:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:29Z" name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc.18be3d24678f1b15 namespace: kserve-ci-e2e-test resourceVersion: "15243" uid: 6358aab3-948d-4027-bfd0-08440eb1ecd4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 337a6fc4-6f5e-4cb7-bf45-5fb46907b6b1 kind: Event lastTimestamp: "2026-07-01T18:15:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:15:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:29Z" name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc.18be3d2467f73718 namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 819661bd-d7dd-41a2-bf71-4443c3b27f09 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 337a6fc4-6f5e-4cb7-bf45-5fb46907b6b1 kind: Event lastTimestamp: "2026-07-01T18:15:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:15:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:29Z" name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc.18be3d2468025e0c namespace: kserve-ci-e2e-test resourceVersion: "15246" uid: b17fdace-81c6-4d19-8e7f-aa3714f0aa6c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 337a6fc4-6f5e-4cb7-bf45-5fb46907b6b1 kind: Event lastTimestamp: "2026-07-01T18:15:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:15:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:29Z" name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc.18be3d246ae58221 namespace: kserve-ci-e2e-test resourceVersion: "15248" uid: b24185b8-017a-43f4-b027-dbf8a40cf60e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 337a6fc4-6f5e-4cb7-bf45-5fb46907b6b1 kind: Event lastTimestamp: "2026-07-01T18:15:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:15:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:15:29Z" name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc.18be3d246b4e3e83 namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: 74a5500c-9613-4be1-a55f-d49c087018cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T18:15:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 337a6fc4-6f5e-4cb7-bf45-5fb46907b6b1 kind: Event lastTimestamp: "2026-07-01T18:16:55Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:15:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:55Z" name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc.18be3d24852c8cd0 namespace: kserve-ci-e2e-test resourceVersion: "16076" uid: 1c73d70e-7c14-4379-bd66-80d7303e2631 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 337a6fc4-6f5e-4cb7-bf45-5fb46907b6b1 kind: Event lastTimestamp: "2026-07-01T18:16:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:54Z" name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc.18be3d384f817d1f namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: 50f776ab-5219-4308-b0d2-fd0bdfb6c368 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 337a6fc4-6f5e-4cb7-bf45-5fb46907b6b1 kind: Event lastTimestamp: "2026-07-01T18:16:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:54Z" name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc.18be3d384f82b0bd namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 1a34e803-63a2-4480-9bae-3f53b8276a7b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 337a6fc4-6f5e-4cb7-bf45-5fb46907b6b1 kind: Event lastTimestamp: "2026-07-01T18:16:55Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:55Z" name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc.18be3d388b66a2ec namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: 044f55c4-af50-40e1-9a33-bab0dc51e0e2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: f44ad80c-97a3-48c4-b91c-eb3c6df534ac kind: Event lastTimestamp: "2026-07-01T18:15:04Z" message: 'Created pod: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf-s96kc' metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:15:04Z" name: isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf.18be3d1e9d0da328 namespace: kserve-ci-e2e-test resourceVersion: "15001" uid: 5dd4a598-dd2e-4256-bdfe-517dcd097c76 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-ebf1b-predictor namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 8863c3cb-e566-41a4-b90f-3ee53d1348ec kind: Event lastTimestamp: "2026-07-01T18:15:04Z" message: Scaled up replica set isvc-xgboost-graph-raw-ebf1b-predictor-56996dd5bf from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:15:04Z" name: isvc-xgboost-graph-raw-ebf1b-predictor.18be3d1e9a41261d namespace: kserve-ci-e2e-test resourceVersion: "14988" uid: 7ade28dc-bb68-4550-bfd7-b747813c0e2a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:15:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-ebf1b-predictor namespace: kserve-ci-e2e-test resourceVersion: "14984" uid: 1318e534-d513-4bdc-bce7-0b1c8d39c9e9 kind: Event lastTimestamp: "2026-07-01T18:16:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:15:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:16:04Z" name: isvc-xgboost-graph-raw-ebf1b-predictor.18be3d2218e3f932 namespace: kserve-ci-e2e-test resourceVersion: "15508" uid: 6c575c0a-aac3-4d32-8649-90339d827615 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:15:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-ebf1b-predictor namespace: kserve-ci-e2e-test resourceVersion: "14984" uid: 1318e534-d513-4bdc-bce7-0b1c8d39c9e9 kind: Event lastTimestamp: "2026-07-01T18:16:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:15:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:16:04Z" name: isvc-xgboost-graph-raw-ebf1b-predictor.18be3d2219408184 namespace: kserve-ci-e2e-test resourceVersion: "15509" uid: fc0d5f7b-9f17-4f82-bed6-5c05ecf73c11 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:16:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-ebf1b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15176" uid: 1318e534-d513-4bdc-bce7-0b1c8d39c9e9 kind: Event lastTimestamp: "2026-07-01T18:16:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:16:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:16:49Z" name: isvc-xgboost-graph-raw-ebf1b-predictor.18be3d3013ff4a2a namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: 9ca20753-0a39-47d2-a120-899f267ea446 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:16:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-ebf1b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15176" uid: 1318e534-d513-4bdc-bce7-0b1c8d39c9e9 kind: Event lastTimestamp: "2026-07-01T18:16:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:16:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:16:49Z" name: isvc-xgboost-graph-raw-ebf1b-predictor.18be3d30145cae7e namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: 927a2a6a-7b8e-4912-ac7e-dba35c5de57f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T18:15:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-ebf1b namespace: kserve-ci-e2e-test resourceVersion: "14947" uid: 74e7f565-45f9-4643-ba23-e7cc5bad433d kind: Event lastTimestamp: "2026-07-01T18:16:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:16:35Z" name: isvc-xgboost-graph-raw-ebf1b.18be3d1e9aa287c7 namespace: kserve-ci-e2e-test resourceVersion: "15764" uid: ee597cf3-e608-4202-926f-ffdc023f0fe1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:15:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-ebf1b namespace: kserve-ci-e2e-test resourceVersion: "14947" uid: 74e7f565-45f9-4643-ba23-e7cc5bad433d kind: Event lastTimestamp: "2026-07-01T18:15:04Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-ebf1b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-ebf1b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:15:04Z" name: isvc-xgboost-graph-raw-ebf1b.18be3d1e9af8f08e namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: 9f64501c-e02d-4227-af1d-1d401d680653 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-ebf1b namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 74e7f565-45f9-4643-ba23-e7cc5bad433d kind: Event lastTimestamp: "2026-07-01T18:16:35Z" message: InferenceService [isvc-xgboost-graph-raw-ebf1b] is Ready metadata: creationTimestamp: "2026-07-01T18:16:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:16:35Z" name: isvc-xgboost-graph-raw-ebf1b.18be3d33e91e58c7 namespace: kserve-ci-e2e-test resourceVersion: "15763" uid: d1db5add-0f6f-4255-9f85-cabbbfd3956f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:16:54.508758Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p namespace: kserve-ci-e2e-test resourceVersion: "16019" uid: 79826905-847d-477d-b67c-25a7f824df7a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p to ip-10-0-132-145.ec2.internal metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:16:54Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p.18be3d384d3f7f21 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 0ec78f65-1c7d-4481-8c4d-3eb253eb345b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-849956d989-wqhnw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:54Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 79826905-847d-477d-b67c-25a7f824df7a kind: Event lastTimestamp: "2026-07-01T18:16:54Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:16:54Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p.18be3d38684b4514 namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 4c652409-afcd-471c-9841-00b81cbe42be reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 79826905-847d-477d-b67c-25a7f824df7a kind: Event lastTimestamp: "2026-07-01T18:16:54Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:54Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p.18be3d3869ccfae2 namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: 3793962b-250e-4bec-9bd4-7f999d94f134 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 79826905-847d-477d-b67c-25a7f824df7a kind: Event lastTimestamp: "2026-07-01T18:16:55Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:55Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p.18be3d386c7d5c8d namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 5fd5ad32-bdde-4e69-9e71-43e16b7faad3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 79826905-847d-477d-b67c-25a7f824df7a kind: Event lastTimestamp: "2026-07-01T18:16:55Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:55Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p.18be3d386cf25d68 namespace: kserve-ci-e2e-test resourceVersion: "16070" uid: 2498148e-1b18-482e-b8f1-5a58ed3d1b31 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 79826905-847d-477d-b67c-25a7f824df7a kind: Event lastTimestamp: "2026-07-01T18:16:58Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-07-01T18:16:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:58Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p.18be3d3957d33926 namespace: kserve-ci-e2e-test resourceVersion: "16118" uid: 21b93230-906d-4372-966f-8f7052556d29 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 79826905-847d-477d-b67c-25a7f824df7a kind: Event lastTimestamp: "2026-07-01T18:16:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:16:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:59Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p.18be3d395b281204 namespace: kserve-ci-e2e-test resourceVersion: "16125" uid: 8ab59167-d906-4371-a6ed-ac74cc97f5db reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 79826905-847d-477d-b67c-25a7f824df7a kind: Event lastTimestamp: "2026-07-01T18:16:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:16:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:59Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p.18be3d395b997da2 namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: da89b6ab-4b73-4cd2-ab71-c1b7dd9c0fcf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 79826905-847d-477d-b67c-25a7f824df7a kind: Event lastTimestamp: "2026-07-01T18:16:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:16:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:59Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p.18be3d395ba4fde3 namespace: kserve-ci-e2e-test resourceVersion: "16131" uid: dc628553-6aef-4067-a63a-363149818b8f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 79826905-847d-477d-b67c-25a7f824df7a kind: Event lastTimestamp: "2026-07-01T18:16:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:16:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:59Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p.18be3d395ee3d956 namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: d1a32273-a4b3-4328-805b-b7de72e055b8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 79826905-847d-477d-b67c-25a7f824df7a kind: Event lastTimestamp: "2026-07-01T18:16:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:16:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:59Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p.18be3d395f570c13 namespace: kserve-ci-e2e-test resourceVersion: "16138" uid: 2da3cec5-b4d2-4a3e-bb3f-c6cdebf04d30 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T18:16:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 79826905-847d-477d-b67c-25a7f824df7a kind: Event lastTimestamp: "2026-07-01T18:18:46Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:16:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:46Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p.18be3d399434e5d8 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 3a672502-7886-4f2d-a624-4938b9c37f46 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 79826905-847d-477d-b67c-25a7f824df7a kind: Event lastTimestamp: "2026-07-01T18:18:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:18:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:44Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p.18be3d51fcc5f1b2 namespace: kserve-ci-e2e-test resourceVersion: "17072" uid: f6c53e9c-f052-4830-bf39-977d34530089 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 79826905-847d-477d-b67c-25a7f824df7a kind: Event lastTimestamp: "2026-07-01T18:18:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:18:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:44Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p.18be3d51fcc680ef namespace: kserve-ci-e2e-test resourceVersion: "17073" uid: 5b7fbc54-79d3-4969-a102-30876831f681 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 79826905-847d-477d-b67c-25a7f824df7a kind: Event lastTimestamp: "2026-07-01T18:18:46Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:18:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:46Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p.18be3d52434d091b namespace: kserve-ci-e2e-test resourceVersion: "17093" uid: c2ea8538-605b-48fa-a62c-6bf9ad278dcb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf namespace: kserve-ci-e2e-test resourceVersion: "16011" uid: 9cfc51fc-22f4-48eb-9376-f17e3d48c0c4 kind: Event lastTimestamp: "2026-07-01T18:16:54Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf-zhn6p' metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:16:54Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf.18be3d384ccb49fa namespace: kserve-ci-e2e-test resourceVersion: "16023" uid: 35bc47b1-acf2-402e-ad8d-097e001942f5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-c9b93-predictor namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: 3490be25-4b72-4e47-833d-49a3743df9f0 kind: Event lastTimestamp: "2026-07-01T18:16:54Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-c9b93-predictor-76bd69bcdf from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:16:54Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor.18be3d384a0cf4e7 namespace: kserve-ci-e2e-test resourceVersion: "16012" uid: 57df6baa-2e67-4773-bc97-8b59f23b0fff reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:17:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-c9b93-predictor namespace: kserve-ci-e2e-test resourceVersion: "16009" uid: cc9b53f1-012a-482a-bb68-7b91369070b2 kind: Event lastTimestamp: "2026-07-01T18:17:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:17:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:17:39Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor.18be3d3bc8818ac4 namespace: kserve-ci-e2e-test resourceVersion: "16453" uid: f1585f30-c393-4fab-91ab-dd5e584e140b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:17:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-c9b93-predictor namespace: kserve-ci-e2e-test resourceVersion: "16009" uid: cc9b53f1-012a-482a-bb68-7b91369070b2 kind: Event lastTimestamp: "2026-07-01T18:17:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:17:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:17:39Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor.18be3d3bc8e33769 namespace: kserve-ci-e2e-test resourceVersion: "16454" uid: 3f1aaa7f-b637-433a-b658-329fdb66f8df reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:17:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-c9b93-predictor namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: cc9b53f1-012a-482a-bb68-7b91369070b2 kind: Event lastTimestamp: "2026-07-01T18:18:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:18:39Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor.18be3d4644f695eb namespace: kserve-ci-e2e-test resourceVersion: "16941" uid: d57702eb-335c-48b9-ae8c-64a12ed08b5e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:17:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-c9b93-predictor namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: cc9b53f1-012a-482a-bb68-7b91369070b2 kind: Event lastTimestamp: "2026-07-01T18:18:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:18:39Z" name: isvc-xgboost-graph-raw-hpa-c9b93-predictor.18be3d464565faf8 namespace: kserve-ci-e2e-test resourceVersion: "16942" uid: 41302ec1-d51d-4e36-a75c-fc07098e4453 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T18:16:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-c9b93 namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: bbc1d635-29b8-4426-8a16-25a583b86156 kind: Event lastTimestamp: "2026-07-01T18:18:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:18:06Z" name: isvc-xgboost-graph-raw-hpa-c9b93.18be3d384a3dd34f namespace: kserve-ci-e2e-test resourceVersion: "16656" uid: dda67ac5-a49e-43ec-b7b3-09d51ccfa5a6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-c9b93 namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: bbc1d635-29b8-4426-8a16-25a583b86156 kind: Event lastTimestamp: "2026-07-01T18:16:54Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-c9b93": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-c9b93": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:16:54Z" name: isvc-xgboost-graph-raw-hpa-c9b93.18be3d384a96e3d2 namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: c0b3876e-5bfd-4b57-bffc-6108c48a31ab reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-c9b93 namespace: kserve-ci-e2e-test resourceVersion: "16654" uid: bbc1d635-29b8-4426-8a16-25a583b86156 kind: Event lastTimestamp: "2026-07-01T18:18:06Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-c9b93] is Ready metadata: creationTimestamp: "2026-07-01T18:18:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:18:06Z" name: isvc-xgboost-graph-raw-hpa-c9b93.18be3d48f8b30d36 namespace: kserve-ci-e2e-test resourceVersion: "16655" uid: 3e83629f-4fd1-4edc-8c3a-eaef882ebe04 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:18:44.717395Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7 namespace: kserve-ci-e2e-test resourceVersion: "17026" uid: 300b0375-12eb-4350-a0ea-45bffd450ae0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7 to ip-10-0-132-145.ec2.internal metadata: creationTimestamp: "2026-07-01T18:18:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:18:44Z" name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7.18be3d51f631d09e namespace: kserve-ci-e2e-test resourceVersion: "17030" uid: b7c6703e-ae70-4bd5-853a-87b5940df387 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-849956d989-wqhnw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:44Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7 namespace: kserve-ci-e2e-test resourceVersion: "17028" uid: 300b0375-12eb-4350-a0ea-45bffd450ae0 kind: Event lastTimestamp: "2026-07-01T18:18:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-raw-dbad7-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:18:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:44Z" name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7.18be3d5200a471b4 namespace: kserve-ci-e2e-test resourceVersion: "17077" uid: 424b1039-83fc-4760-9719-26f9a01907cd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:45Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7 namespace: kserve-ci-e2e-test resourceVersion: "17039" uid: 300b0375-12eb-4350-a0ea-45bffd450ae0 kind: Event lastTimestamp: "2026-07-01T18:18:45Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:18:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:18:45Z" name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7.18be3d5233aafb54 namespace: kserve-ci-e2e-test resourceVersion: "17088" uid: 2ed57c1a-4398-476e-b501-3748e571fe7c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7 namespace: kserve-ci-e2e-test resourceVersion: "17028" uid: 300b0375-12eb-4350-a0ea-45bffd450ae0 kind: Event lastTimestamp: "2026-07-01T18:18:45Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-01T18:18:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:45Z" name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7.18be3d5234f6cf7f namespace: kserve-ci-e2e-test resourceVersion: "17090" uid: 67b69411-f231-4144-9f51-f99398ec9136 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7 namespace: kserve-ci-e2e-test resourceVersion: "17028" uid: 300b0375-12eb-4350-a0ea-45bffd450ae0 kind: Event lastTimestamp: "2026-07-01T18:18:46Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.202s (1.202s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-01T18:18:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:46Z" name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7.18be3d527c9e27be namespace: kserve-ci-e2e-test resourceVersion: "17102" uid: e887c31a-f1a9-491d-99d0-71b9a7ccebd9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7 namespace: kserve-ci-e2e-test resourceVersion: "17028" uid: 300b0375-12eb-4350-a0ea-45bffd450ae0 kind: Event lastTimestamp: "2026-07-01T18:18:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:47Z" name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7.18be3d527fd49af5 namespace: kserve-ci-e2e-test resourceVersion: "17105" uid: 13452c1e-18fd-4667-ab5e-84d5eb02f4ff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7 namespace: kserve-ci-e2e-test resourceVersion: "17028" uid: 300b0375-12eb-4350-a0ea-45bffd450ae0 kind: Event lastTimestamp: "2026-07-01T18:18:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:47Z" name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7.18be3d5280501501 namespace: kserve-ci-e2e-test resourceVersion: "17106" uid: 389ef33c-6d13-4cf5-8a0c-f7defbe5157b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7 namespace: kserve-ci-e2e-test resourceVersion: "17028" uid: 300b0375-12eb-4350-a0ea-45bffd450ae0 kind: Event lastTimestamp: "2026-07-01T18:18:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:47Z" name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7.18be3d52805a2d56 namespace: kserve-ci-e2e-test resourceVersion: "17107" uid: 6286dee3-7184-449d-9677-682e70262baa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7 namespace: kserve-ci-e2e-test resourceVersion: "17028" uid: 300b0375-12eb-4350-a0ea-45bffd450ae0 kind: Event lastTimestamp: "2026-07-01T18:18:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:47Z" name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7.18be3d528474a9bd namespace: kserve-ci-e2e-test resourceVersion: "17109" uid: 3f7e955e-46eb-4406-820f-6681dca02753 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7 namespace: kserve-ci-e2e-test resourceVersion: "17028" uid: 300b0375-12eb-4350-a0ea-45bffd450ae0 kind: Event lastTimestamp: "2026-07-01T18:18:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:47Z" name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7.18be3d5284f20907 namespace: kserve-ci-e2e-test resourceVersion: "17110" uid: 756bb656-5ac4-47a5-99f5-cda0b0c66c9c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7 namespace: kserve-ci-e2e-test resourceVersion: "17028" uid: 300b0375-12eb-4350-a0ea-45bffd450ae0 kind: Event lastTimestamp: "2026-07-01T18:20:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:40Z" name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7.18be3d6cd5ee0155 namespace: kserve-ci-e2e-test resourceVersion: "18087" uid: df7bff6c-c571-49d0-b0ec-715231c34de9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:20:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7 namespace: kserve-ci-e2e-test resourceVersion: "17028" uid: 300b0375-12eb-4350-a0ea-45bffd450ae0 kind: Event lastTimestamp: "2026-07-01T18:20:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:20:40Z" name: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7.18be3d6cd5ef3ffd namespace: kserve-ci-e2e-test resourceVersion: "18091" uid: ab332424-06f1-43ff-a3a1-42abfb59946f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-dbad7-predictor-94b4c95bd namespace: kserve-ci-e2e-test resourceVersion: "17014" uid: e35e69ab-a2c6-443e-a1a0-45bbaca24a1f kind: Event lastTimestamp: "2026-07-01T18:18:44Z" message: 'Created pod: message-dumper-raw-dbad7-predictor-94b4c95bd-mgtg7' metadata: creationTimestamp: "2026-07-01T18:18:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:18:44Z" name: message-dumper-raw-dbad7-predictor-94b4c95bd.18be3d51f5a323af namespace: kserve-ci-e2e-test resourceVersion: "17027" uid: cf93a0e7-a499-4011-891a-ff8cfda8ec47 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-dbad7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17012" uid: 05cb5789-9d95-4165-b996-f3eb65ef9542 kind: Event lastTimestamp: "2026-07-01T18:18:44Z" message: Scaled up replica set message-dumper-raw-dbad7-predictor-94b4c95bd from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:18:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:18:44Z" name: message-dumper-raw-dbad7-predictor.18be3d51f29d2a0f namespace: kserve-ci-e2e-test resourceVersion: "17017" uid: f6b03b40-8b34-49d9-bcc7-a583c386b1ee reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-dbad7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17013" uid: f09c2605-e449-4d42-ba92-dac272f26681 kind: Event lastTimestamp: "2026-07-01T18:18:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:18:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:18:59Z" name: message-dumper-raw-dbad7-predictor.18be3d55711dda7f namespace: kserve-ci-e2e-test resourceVersion: "17213" uid: f8a15360-d7b2-4104-8972-0c34114df854 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-dbad7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17013" uid: f09c2605-e449-4d42-ba92-dac272f26681 kind: Event lastTimestamp: "2026-07-01T18:18:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:18:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:18:59Z" name: message-dumper-raw-dbad7-predictor.18be3d55717fcfa4 namespace: kserve-ci-e2e-test resourceVersion: "17215" uid: a2999da4-9eb7-446d-b75a-6f54f366de5c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-dbad7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17214" uid: f09c2605-e449-4d42-ba92-dac272f26681 kind: Event lastTimestamp: "2026-07-01T18:19:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:19:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:19:14Z" name: message-dumper-raw-dbad7-predictor.18be3d58f0756ff3 namespace: kserve-ci-e2e-test resourceVersion: "17390" uid: a557a185-fcaf-4071-954e-834e27e927cc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:19:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-dbad7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17214" uid: f09c2605-e449-4d42-ba92-dac272f26681 kind: Event lastTimestamp: "2026-07-01T18:19:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:19:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:19:14Z" name: message-dumper-raw-dbad7-predictor.18be3d58f0dc9acf namespace: kserve-ci-e2e-test resourceVersion: "17392" uid: d4a0248d-f130-465c-9fc7-2cf36ae57809 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T18:18:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-dbad7 namespace: kserve-ci-e2e-test resourceVersion: "17005" uid: be2b079d-aaf4-4693-8173-30b75f41f8e7 kind: Event lastTimestamp: "2026-07-01T18:18:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:18:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:18:54Z" name: message-dumper-raw-dbad7.18be3d51f2d2d7b0 namespace: kserve-ci-e2e-test resourceVersion: "17176" uid: d6094e95-c598-4c03-87bf-1e14b7882e4f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-dbad7 namespace: kserve-ci-e2e-test resourceVersion: "17005" uid: be2b079d-aaf4-4693-8173-30b75f41f8e7 kind: Event lastTimestamp: "2026-07-01T18:18:44Z" message: 'Failed to update status for InferenceService "message-dumper-raw-dbad7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-dbad7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:18:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:18:44Z" name: message-dumper-raw-dbad7.18be3d51f32e9ac7 namespace: kserve-ci-e2e-test resourceVersion: "17022" uid: 0cdee203-e6f4-4643-ba8d-97818af1ef00 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-dbad7 namespace: kserve-ci-e2e-test resourceVersion: "17174" uid: be2b079d-aaf4-4693-8173-30b75f41f8e7 kind: Event lastTimestamp: "2026-07-01T18:18:54Z" message: InferenceService [message-dumper-raw-dbad7] is Ready metadata: creationTimestamp: "2026-07-01T18:18:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:18:54Z" name: message-dumper-raw-dbad7.18be3d54432feb01 namespace: kserve-ci-e2e-test resourceVersion: "17175" uid: 8408aa78-cc8a-44c1-af86-fab88552ee26 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:16:44.147967Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-ebf1b-8479fd844c-d2m5f namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 49874f6a-d4fc-4430-936a-eb9390acb053 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-ebf1b-8479fd844c-d2m5f to ip-10-0-136-131.ec2.internal metadata: creationTimestamp: "2026-07-01T18:16:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:16:44Z" name: model-chainer-raw-ebf1b-8479fd844c-d2m5f.18be3d35e3b259b8 namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: 71dfb6d9-4b15-4593-82da-d33a8326d85a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-849956d989-wqhnw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:44Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-ebf1b-8479fd844c-d2m5f namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 49874f6a-d4fc-4430-936a-eb9390acb053 kind: Event lastTimestamp: "2026-07-01T18:16:44Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:16:44Z" name: model-chainer-raw-ebf1b-8479fd844c-d2m5f.18be3d35fcc9022c namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: e1199d0b-2609-40db-a2b0-2aa5951ab5b5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-ebf1b} kind: Pod name: model-chainer-raw-ebf1b-8479fd844c-d2m5f namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 49874f6a-d4fc-4430-936a-eb9390acb053 kind: Event lastTimestamp: "2026-07-01T18:16:44Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:4d6fcc0e5d3b23dc1e66af36b5e4c4d4c9974f1880f849ee4e4508e9f5fd89ff" metadata: creationTimestamp: "2026-07-01T18:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:44Z" name: model-chainer-raw-ebf1b-8479fd844c-d2m5f.18be3d35fdeec801 namespace: kserve-ci-e2e-test resourceVersion: "15856" uid: 8e3767d6-c17d-4cd1-bebb-08badbc9afa3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-131.ec2.internal source: component: kubelet host: ip-10-0-136-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-ebf1b} kind: Pod name: model-chainer-raw-ebf1b-8479fd844c-d2m5f namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 49874f6a-d4fc-4430-936a-eb9390acb053 kind: Event lastTimestamp: "2026-07-01T18:16:47Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:4d6fcc0e5d3b23dc1e66af36b5e4c4d4c9974f1880f849ee4e4508e9f5fd89ff" in 2.448s (2.448s including waiting). Image size: 218070118 bytes.' metadata: creationTimestamp: "2026-07-01T18:16:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:47Z" name: model-chainer-raw-ebf1b-8479fd844c-d2m5f.18be3d368fd92a1d namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: b70e6f29-51dd-4808-bfb6-dde6688a1041 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-131.ec2.internal source: component: kubelet host: ip-10-0-136-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-ebf1b} kind: Pod name: model-chainer-raw-ebf1b-8479fd844c-d2m5f namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 49874f6a-d4fc-4430-936a-eb9390acb053 kind: Event lastTimestamp: "2026-07-01T18:16:47Z" message: 'Created container: model-chainer-raw-ebf1b' metadata: creationTimestamp: "2026-07-01T18:16:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:47Z" name: model-chainer-raw-ebf1b-8479fd844c-d2m5f.18be3d3692906991 namespace: kserve-ci-e2e-test resourceVersion: "15871" uid: 6dcd6516-ce52-4383-9ad2-48093211f42e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-131.ec2.internal source: component: kubelet host: ip-10-0-136-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-ebf1b} kind: Pod name: model-chainer-raw-ebf1b-8479fd844c-d2m5f namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 49874f6a-d4fc-4430-936a-eb9390acb053 kind: Event lastTimestamp: "2026-07-01T18:16:47Z" message: Started container model-chainer-raw-ebf1b metadata: creationTimestamp: "2026-07-01T18:16:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:47Z" name: model-chainer-raw-ebf1b-8479fd844c-d2m5f.18be3d3692fcdc7a namespace: kserve-ci-e2e-test resourceVersion: "15873" uid: abab84d7-c9a7-4f29-9f42-33d8e34bb0d3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-131.ec2.internal source: component: kubelet host: ip-10-0-136-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-ebf1b} kind: Pod name: model-chainer-raw-ebf1b-8479fd844c-d2m5f namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 49874f6a-d4fc-4430-936a-eb9390acb053 kind: Event lastTimestamp: "2026-07-01T18:16:54Z" message: Stopping container model-chainer-raw-ebf1b metadata: creationTimestamp: "2026-07-01T18:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:16:54Z" name: model-chainer-raw-ebf1b-8479fd844c-d2m5f.18be3d383b39b6f5 namespace: kserve-ci-e2e-test resourceVersion: "15946" uid: 7caee48c-f77c-490e-8772-a0a640d89475 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-131.ec2.internal source: component: kubelet host: ip-10-0-136-131.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:16:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-ebf1b} kind: Pod name: model-chainer-raw-ebf1b-8479fd844c-d2m5f namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 49874f6a-d4fc-4430-936a-eb9390acb053 kind: Event lastTimestamp: "2026-07-01T18:17:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:16:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:17:23Z" name: model-chainer-raw-ebf1b-8479fd844c-d2m5f.18be3d393db444a2 namespace: kserve-ci-e2e-test resourceVersion: "16332" uid: fd25d235-1762-4fef-81a8-64a1c7dfdc38 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-131.ec2.internal source: component: kubelet host: ip-10-0-136-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-ebf1b-8479fd844c namespace: kserve-ci-e2e-test resourceVersion: "15831" uid: 86b30acb-b705-4954-89cc-b57e05718ff4 kind: Event lastTimestamp: "2026-07-01T18:16:44Z" message: 'Created pod: model-chainer-raw-ebf1b-8479fd844c-d2m5f' metadata: creationTimestamp: "2026-07-01T18:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:16:44Z" name: model-chainer-raw-ebf1b-8479fd844c.18be3d35e3394ddc namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: c0d4d8ad-90d1-4170-9664-17af5eabf609 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-ebf1b namespace: kserve-ci-e2e-test resourceVersion: "15829" uid: 777a6160-3f1a-40de-8423-0af44981970e kind: Event lastTimestamp: "2026-07-01T18:16:44Z" message: Scaled up replica set model-chainer-raw-ebf1b-8479fd844c from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:16:44Z" name: model-chainer-raw-ebf1b.18be3d35e2047506 namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: 9cf67772-b133-4486-b480-404150075cae reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:16:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-ebf1b namespace: kserve-ci-e2e-test resourceVersion: "15845" uid: 035b8386-ca6e-4302-8478-bb862b3e9098 kind: Event lastTimestamp: "2026-07-01T18:16:44Z" message: InferenceGraph [model-chainer-raw-ebf1b] is Ready metadata: creationTimestamp: "2026-07-01T18:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:16:44Z" name: model-chainer-raw-ebf1b.18be3d35e6ee2b28 namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: 605c18dc-ca5c-48e9-8b80-99f5b3a62660 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:18:34.450872Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-c9b93-67b958dfc9-978rt namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: f45bb1be-6840-467a-9826-19699bd86c14 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-c9b93-67b958dfc9-978rt to ip-10-0-136-131.ec2.internal metadata: creationTimestamp: "2026-07-01T18:18:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:18:34Z" name: model-chainer-raw-hpa-c9b93-67b958dfc9-978rt.18be3d4f92432dc3 namespace: kserve-ci-e2e-test resourceVersion: "16883" uid: cd7662e9-228a-4558-a772-fef2ed16ba08 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-849956d989-wqhnw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:34Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-c9b93-67b958dfc9-978rt namespace: kserve-ci-e2e-test resourceVersion: "16880" uid: f45bb1be-6840-467a-9826-19699bd86c14 kind: Event lastTimestamp: "2026-07-01T18:18:34Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-c9b93-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:18:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:34Z" name: model-chainer-raw-hpa-c9b93-67b958dfc9-978rt.18be3d4f9e456a81 namespace: kserve-ci-e2e-test resourceVersion: "16894" uid: 9a5a7843-4862-4c18-aa02-30118354a6a2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-131.ec2.internal source: component: kubelet host: ip-10-0-136-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:35Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-c9b93-67b958dfc9-978rt namespace: kserve-ci-e2e-test resourceVersion: "16887" uid: f45bb1be-6840-467a-9826-19699bd86c14 kind: Event lastTimestamp: "2026-07-01T18:18:35Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:18:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:18:35Z" name: model-chainer-raw-hpa-c9b93-67b958dfc9-978rt.18be3d4fcf3109f6 namespace: kserve-ci-e2e-test resourceVersion: "16904" uid: 3b66e2b4-d492-44bc-9589-5c5245751b82 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-c9b93} kind: Pod name: model-chainer-raw-hpa-c9b93-67b958dfc9-978rt namespace: kserve-ci-e2e-test resourceVersion: "16880" uid: f45bb1be-6840-467a-9826-19699bd86c14 kind: Event lastTimestamp: "2026-07-01T18:18:35Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4d6fcc0e5d3b23dc1e66af36b5e4c4d4c9974f1880f849ee4e4508e9f5fd89ff" already present on machine metadata: creationTimestamp: "2026-07-01T18:18:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:35Z" name: model-chainer-raw-hpa-c9b93-67b958dfc9-978rt.18be3d4fd05cb88e namespace: kserve-ci-e2e-test resourceVersion: "16907" uid: e1421dc7-4e85-4a69-8d84-7e29a6d1c7f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-131.ec2.internal source: component: kubelet host: ip-10-0-136-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-c9b93} kind: Pod name: model-chainer-raw-hpa-c9b93-67b958dfc9-978rt namespace: kserve-ci-e2e-test resourceVersion: "16880" uid: f45bb1be-6840-467a-9826-19699bd86c14 kind: Event lastTimestamp: "2026-07-01T18:18:35Z" message: 'Created container: model-chainer-raw-hpa-c9b93' metadata: creationTimestamp: "2026-07-01T18:18:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:35Z" name: model-chainer-raw-hpa-c9b93-67b958dfc9-978rt.18be3d4fd35f46cf namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 9b88e564-80e3-480d-a39d-a3d6240b1d4c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-131.ec2.internal source: component: kubelet host: ip-10-0-136-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-c9b93} kind: Pod name: model-chainer-raw-hpa-c9b93-67b958dfc9-978rt namespace: kserve-ci-e2e-test resourceVersion: "16880" uid: f45bb1be-6840-467a-9826-19699bd86c14 kind: Event lastTimestamp: "2026-07-01T18:18:35Z" message: Started container model-chainer-raw-hpa-c9b93 metadata: creationTimestamp: "2026-07-01T18:18:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:35Z" name: model-chainer-raw-hpa-c9b93-67b958dfc9-978rt.18be3d4fd3f975ba namespace: kserve-ci-e2e-test resourceVersion: "16909" uid: 9ad78051-9519-4f2f-b5c6-507d022f77bb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-131.ec2.internal source: component: kubelet host: ip-10-0-136-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-c9b93} kind: Pod name: model-chainer-raw-hpa-c9b93-67b958dfc9-978rt namespace: kserve-ci-e2e-test resourceVersion: "16880" uid: f45bb1be-6840-467a-9826-19699bd86c14 kind: Event lastTimestamp: "2026-07-01T18:18:44Z" message: Stopping container model-chainer-raw-hpa-c9b93 metadata: creationTimestamp: "2026-07-01T18:18:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:18:44Z" name: model-chainer-raw-hpa-c9b93-67b958dfc9-978rt.18be3d51e957ec6d namespace: kserve-ci-e2e-test resourceVersion: "17000" uid: 1a07e654-feb0-4d77-9c83-e309bf0f9be4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-131.ec2.internal source: component: kubelet host: ip-10-0-136-131.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:18:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-c9b93} kind: Pod name: model-chainer-raw-hpa-c9b93-67b958dfc9-978rt namespace: kserve-ci-e2e-test resourceVersion: "16880" uid: f45bb1be-6840-467a-9826-19699bd86c14 kind: Event lastTimestamp: "2026-07-01T18:19:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:18:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:19:11Z" name: model-chainer-raw-hpa-c9b93-67b958dfc9-978rt.18be3d5279df41a5 namespace: kserve-ci-e2e-test resourceVersion: "17371" uid: f27da823-dd1c-41f4-9f15-39bde6b5d70f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-131.ec2.internal source: component: kubelet host: ip-10-0-136-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-c9b93-67b958dfc9 namespace: kserve-ci-e2e-test resourceVersion: "16875" uid: 0394e00b-bb3f-4c77-9f64-0f13b501f688 kind: Event lastTimestamp: "2026-07-01T18:18:34Z" message: 'Created pod: model-chainer-raw-hpa-c9b93-67b958dfc9-978rt' metadata: creationTimestamp: "2026-07-01T18:18:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:18:34Z" name: model-chainer-raw-hpa-c9b93-67b958dfc9.18be3d4f91e479e0 namespace: kserve-ci-e2e-test resourceVersion: "16881" uid: f7fef5c4-3e0f-4c6f-8def-b3fa700fffd7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-c9b93 namespace: kserve-ci-e2e-test resourceVersion: "16873" uid: 40a0d322-dc0a-4272-8a27-eec9e7c54b36 kind: Event lastTimestamp: "2026-07-01T18:18:34Z" message: Scaled up replica set model-chainer-raw-hpa-c9b93-67b958dfc9 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:18:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:18:34Z" name: model-chainer-raw-hpa-c9b93.18be3d4f90854dbd namespace: kserve-ci-e2e-test resourceVersion: "16876" uid: 995660fc-bc4c-4d04-8561-e7a070201db0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:18:34Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-c9b93 namespace: kserve-ci-e2e-test resourceVersion: "16889" uid: c0c102ec-e838-4556-ad89-68bd491025e6 kind: Event lastTimestamp: "2026-07-01T18:18:34Z" message: InferenceGraph [model-chainer-raw-hpa-c9b93] is Ready metadata: creationTimestamp: "2026-07-01T18:18:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:18:34Z" name: model-chainer-raw-hpa-c9b93.18be3d4f95e8e8d4 namespace: kserve-ci-e2e-test resourceVersion: "16891" uid: a1f6c5cb-5d7e-461d-8673-5d2f3ab7a4d3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:25:39.018067Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq namespace: kserve-ci-e2e-test resourceVersion: "20540" uid: f8ac156b-a0b7-4a6d-a3e1-9c26417aadcb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-20421-predictor-58f54f4cf9-czsvq to ip-10-0-132-145.ec2.internal metadata: creationTimestamp: "2026-07-01T18:25:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:25:39Z" name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq.18be3db26c7077a4 namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: 7f3a9444-9db7-4543-86f0-7ec1ae468d48 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-849956d989-wqhnw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:39Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq namespace: kserve-ci-e2e-test resourceVersion: "20550" uid: f8ac156b-a0b7-4a6d-a3e1-9c26417aadcb kind: Event lastTimestamp: "2026-07-01T18:25:39Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:25:39Z" name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq.18be3db2867870f4 namespace: kserve-ci-e2e-test resourceVersion: "20559" uid: 2503482f-f305-40a0-a1dd-51336a03d2fa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: f8ac156b-a0b7-4a6d-a3e1-9c26417aadcb kind: Event lastTimestamp: "2026-07-01T18:25:39Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:39Z" name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq.18be3db287de6cd0 namespace: kserve-ci-e2e-test resourceVersion: "20561" uid: e73b2905-1cff-48f5-b66a-371355ab2c9c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: f8ac156b-a0b7-4a6d-a3e1-9c26417aadcb kind: Event lastTimestamp: "2026-07-01T18:25:39Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:39Z" name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq.18be3db28b5fb892 namespace: kserve-ci-e2e-test resourceVersion: "20564" uid: 756a9fb6-592f-45e1-a544-50407337fc70 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: f8ac156b-a0b7-4a6d-a3e1-9c26417aadcb kind: Event lastTimestamp: "2026-07-01T18:25:39Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:39Z" name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq.18be3db28bde1dd0 namespace: kserve-ci-e2e-test resourceVersion: "20565" uid: 7d287af0-685e-409f-b10e-4c102077ff66 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: f8ac156b-a0b7-4a6d-a3e1-9c26417aadcb kind: Event lastTimestamp: "2026-07-01T18:25:43Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:25:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:43Z" name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq.18be3db37361e953 namespace: kserve-ci-e2e-test resourceVersion: "20597" uid: 1bc0063c-d388-47c2-bd26-7b1ffb1a3cff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: f8ac156b-a0b7-4a6d-a3e1-9c26417aadcb kind: Event lastTimestamp: "2026-07-01T18:25:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:25:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:43Z" name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq.18be3db376588cea namespace: kserve-ci-e2e-test resourceVersion: "20599" uid: b7605e14-2df5-4873-b1c1-9cbad7120d05 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: f8ac156b-a0b7-4a6d-a3e1-9c26417aadcb kind: Event lastTimestamp: "2026-07-01T18:25:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:25:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:43Z" name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq.18be3db376cf1220 namespace: kserve-ci-e2e-test resourceVersion: "20603" uid: 7d1da804-6662-496c-9418-8ca05e3fa376 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: f8ac156b-a0b7-4a6d-a3e1-9c26417aadcb kind: Event lastTimestamp: "2026-07-01T18:25:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:25:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:43Z" name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq.18be3db376d8bcf6 namespace: kserve-ci-e2e-test resourceVersion: "20604" uid: c27f19bd-ff90-406b-8c03-e3ef35236775 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: f8ac156b-a0b7-4a6d-a3e1-9c26417aadcb kind: Event lastTimestamp: "2026-07-01T18:25:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:25:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:43Z" name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq.18be3db37af321fa namespace: kserve-ci-e2e-test resourceVersion: "20606" uid: 3020fef2-427b-4fd0-bf5b-75134cf2999b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: f8ac156b-a0b7-4a6d-a3e1-9c26417aadcb kind: Event lastTimestamp: "2026-07-01T18:25:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:25:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:25:43Z" name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq.18be3db37b72fcf5 namespace: kserve-ci-e2e-test resourceVersion: "20608" uid: 7cd35ce6-571c-488d-a115-9fb564798d91 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T18:25:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: f8ac156b-a0b7-4a6d-a3e1-9c26417aadcb kind: Event lastTimestamp: "2026-07-01T18:27:10Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:25:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:10Z" name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq.18be3db3af74e04f namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: 6ab79bf4-e03c-4499-97ef-d0dc3ea85735 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: f8ac156b-a0b7-4a6d-a3e1-9c26417aadcb kind: Event lastTimestamp: "2026-07-01T18:27:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:09Z" name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq.18be3dc76b56eb78 namespace: kserve-ci-e2e-test resourceVersion: "21238" uid: 7071760f-8f95-4feb-9884-2113bf15a805 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: f8ac156b-a0b7-4a6d-a3e1-9c26417aadcb kind: Event lastTimestamp: "2026-07-01T18:27:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:09Z" name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq.18be3dc76b581813 namespace: kserve-ci-e2e-test resourceVersion: "21239" uid: d1ef6276-0e18-484c-8f46-23f7790b59dd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: f8ac156b-a0b7-4a6d-a3e1-9c26417aadcb kind: Event lastTimestamp: "2026-07-01T18:27:10Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:10Z" name: raw-sklearn-20421-predictor-58f54f4cf9-czsvq.18be3dc7b5b8052a namespace: kserve-ci-e2e-test resourceVersion: "21246" uid: ac404767-96d6-419f-9517-c4d3ea54acf0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-20421-predictor-58f54f4cf9 namespace: kserve-ci-e2e-test resourceVersion: "20531" uid: c41d9507-5c0f-4aa2-bfd6-b524d09bc7f6 kind: Event lastTimestamp: "2026-07-01T18:25:39Z" message: 'Created pod: raw-sklearn-20421-predictor-58f54f4cf9-czsvq' metadata: creationTimestamp: "2026-07-01T18:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:25:39Z" name: raw-sklearn-20421-predictor-58f54f4cf9.18be3db26bff55bc namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: 856814e5-5bde-41c5-af03-75e9dbd326bd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-20421-predictor namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: 87958928-93d3-47c8-be67-1e642b5ee6f7 kind: Event lastTimestamp: "2026-07-01T18:25:38Z" message: Scaled up replica set raw-sklearn-20421-predictor-58f54f4cf9 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:25:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:25:38Z" name: raw-sklearn-20421-predictor.18be3db269193561 namespace: kserve-ci-e2e-test resourceVersion: "20534" uid: 15036f75-79e5-4351-aad4-6784de8c9d47 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:25:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-20421-predictor namespace: kserve-ci-e2e-test resourceVersion: "20530" uid: d480eac6-506d-4e1b-a105-77b92e26ac27 kind: Event lastTimestamp: "2026-07-01T18:26:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:25:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:26:24Z" name: raw-sklearn-20421-predictor.18be3db5e7916fde namespace: kserve-ci-e2e-test resourceVersion: "20893" uid: 7e898c99-be23-4fae-98e6-c897a75ffd92 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:25:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-20421-predictor namespace: kserve-ci-e2e-test resourceVersion: "20530" uid: d480eac6-506d-4e1b-a105-77b92e26ac27 kind: Event lastTimestamp: "2026-07-01T18:26:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:25:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:26:24Z" name: raw-sklearn-20421-predictor.18be3db5e7ff0707 namespace: kserve-ci-e2e-test resourceVersion: "20894" uid: 89bca5f5-7f67-4954-ae63-1339ad0159e5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:26:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-20421-predictor namespace: kserve-ci-e2e-test resourceVersion: "20682" uid: d480eac6-506d-4e1b-a105-77b92e26ac27 kind: Event lastTimestamp: "2026-07-01T18:27:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:27:09Z" name: raw-sklearn-20421-predictor.18be3dc0649ccba5 namespace: kserve-ci-e2e-test resourceVersion: "21220" uid: 56def3ca-a13d-4873-98dd-dbd331236fb5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:26:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-20421-predictor namespace: kserve-ci-e2e-test resourceVersion: "20682" uid: d480eac6-506d-4e1b-a105-77b92e26ac27 kind: Event lastTimestamp: "2026-07-01T18:27:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:27:09Z" name: raw-sklearn-20421-predictor.18be3dc0650896f3 namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 67cf5423-f77e-47c0-8929-4769ace7fb3c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T18:25:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-20421 namespace: kserve-ci-e2e-test resourceVersion: "20520" uid: 08b48307-919d-494e-97a3-1ee7d9b1bef9 kind: Event lastTimestamp: "2026-07-01T18:27:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:25:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:27:00Z" name: raw-sklearn-20421.18be3db2691cecfd namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: f82a406b-694e-4024-93c4-ff45e4ea06f3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:25:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-20421 namespace: kserve-ci-e2e-test resourceVersion: "20520" uid: 08b48307-919d-494e-97a3-1ee7d9b1bef9 kind: Event lastTimestamp: "2026-07-01T18:25:38Z" message: 'Failed to update status for InferenceService "raw-sklearn-20421": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-20421": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:25:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:25:38Z" name: raw-sklearn-20421.18be3db2696a87fe namespace: kserve-ci-e2e-test resourceVersion: "20536" uid: e9e17499-546b-4d4f-8716-29bccd715b14 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-20421 namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 08b48307-919d-494e-97a3-1ee7d9b1bef9 kind: Event lastTimestamp: "2026-07-01T18:27:00Z" message: InferenceService [raw-sklearn-20421] is Ready metadata: creationTimestamp: "2026-07-01T18:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:27:00Z" name: raw-sklearn-20421.18be3dc567cf523e namespace: kserve-ci-e2e-test resourceVersion: "21161" uid: fd1e4ac1-8957-48f9-b25a-4d9ada1a34ec reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:27:14.336341Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6 namespace: kserve-ci-e2e-test resourceVersion: "21297" uid: 17063895-46d6-4356-b1f9-7ad387266e1a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6 to ip-10-0-132-145.ec2.internal metadata: creationTimestamp: "2026-07-01T18:27:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:27:14Z" name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6.18be3dc89dd9e21c namespace: kserve-ci-e2e-test resourceVersion: "21301" uid: 760417ca-75f7-4694-97bd-9ca6900b5496 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-849956d989-wqhnw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:14Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 17063895-46d6-4356-b1f9-7ad387266e1a kind: Event lastTimestamp: "2026-07-01T18:27:14Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-runtime-10c8f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:27:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:14Z" name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6.18be3dc8a7c998a3 namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: eecdf089-cac5-4c93-8ebf-9ef1292954ed reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:15Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6 namespace: kserve-ci-e2e-test resourceVersion: "21305" uid: 17063895-46d6-4356-b1f9-7ad387266e1a kind: Event lastTimestamp: "2026-07-01T18:27:15Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:27:15Z" name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6.18be3dc8db765df7 namespace: kserve-ci-e2e-test resourceVersion: "21319" uid: 31d91099-6e9d-478e-86f9-18bbc77f3f4e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 17063895-46d6-4356-b1f9-7ad387266e1a kind: Event lastTimestamp: "2026-07-01T18:27:15Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:d9a53871119e5163dc10506afed26a0cdd4198fd2315ea323b1e9fe09949b475" already present on machine metadata: creationTimestamp: "2026-07-01T18:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:15Z" name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6.18be3dc8dce6ad8c namespace: kserve-ci-e2e-test resourceVersion: "21321" uid: ce8802b7-6adc-44d3-9f01-b5984108f106 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 17063895-46d6-4356-b1f9-7ad387266e1a kind: Event lastTimestamp: "2026-07-01T18:27:15Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:15Z" name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6.18be3dc8e01a916f namespace: kserve-ci-e2e-test resourceVersion: "21323" uid: edade7c9-20bb-4713-9fe9-0186120789c9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 17063895-46d6-4356-b1f9-7ad387266e1a kind: Event lastTimestamp: "2026-07-01T18:27:15Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:15Z" name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6.18be3dc8e092dd0e namespace: kserve-ci-e2e-test resourceVersion: "21324" uid: 9141d108-2fbd-4b5a-98bf-062f405a5a80 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 17063895-46d6-4356-b1f9-7ad387266e1a kind: Event lastTimestamp: "2026-07-01T18:27:19Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-01T18:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:19Z" name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6.18be3dc9e7054847 namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: 96d2a983-f5eb-48be-8003-ef35f433ac3e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 17063895-46d6-4356-b1f9-7ad387266e1a kind: Event lastTimestamp: "2026-07-01T18:27:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:19Z" name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6.18be3dc9e9ee8690 namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 952c62b2-02ed-4830-beef-9dfebe6620c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 17063895-46d6-4356-b1f9-7ad387266e1a kind: Event lastTimestamp: "2026-07-01T18:27:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:19Z" name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6.18be3dc9ea63dc48 namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: cbd9ada4-685a-4b01-87a4-5703b3fa98de reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 17063895-46d6-4356-b1f9-7ad387266e1a kind: Event lastTimestamp: "2026-07-01T18:27:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:19Z" name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6.18be3dc9ea6d2124 namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: af07b09f-cb5f-4944-bbf9-63ad3607214b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 17063895-46d6-4356-b1f9-7ad387266e1a kind: Event lastTimestamp: "2026-07-01T18:27:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:19Z" name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6.18be3dc9ed4cc8fd namespace: kserve-ci-e2e-test resourceVersion: "21369" uid: 43339c60-c973-4b76-8e56-2db891cd547f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 17063895-46d6-4356-b1f9-7ad387266e1a kind: Event lastTimestamp: "2026-07-01T18:27:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:27:19Z" name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6.18be3dc9edc490f8 namespace: kserve-ci-e2e-test resourceVersion: "21371" uid: e698c3f0-1cee-4a34-85ea-0212b537a7b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T18:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 17063895-46d6-4356-b1f9-7ad387266e1a kind: Event lastTimestamp: "2026-07-01T18:28:31Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:28:31Z" name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6.18be3dca23114c1d namespace: kserve-ci-e2e-test resourceVersion: "21883" uid: 67e53ec5-7135-4dd1-9027-a93927805b26 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:28:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 17063895-46d6-4356-b1f9-7ad387266e1a kind: Event lastTimestamp: "2026-07-01T18:28:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:28:44Z" name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6.18be3ddd98481ab6 namespace: kserve-ci-e2e-test resourceVersion: "21997" uid: 48380b0b-189e-4d63-8032-46dc8a33d420 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:28:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 17063895-46d6-4356-b1f9-7ad387266e1a kind: Event lastTimestamp: "2026-07-01T18:28:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:28:44Z" name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6.18be3ddd9848bef3 namespace: kserve-ci-e2e-test resourceVersion: "21998" uid: 8b04ec28-978b-46f9-93bb-a9ba8cdc3ebc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:28:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 17063895-46d6-4356-b1f9-7ad387266e1a kind: Event lastTimestamp: "2026-07-01T18:28:46Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:28:46Z" name: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6.18be3dde295426f2 namespace: kserve-ci-e2e-test resourceVersion: "22015" uid: 152d3ebf-9e4c-4ea7-aa18-f81f09770362 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-145.ec2.internal source: component: kubelet host: ip-10-0-132-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-10c8f-predictor-649b8c799f namespace: kserve-ci-e2e-test resourceVersion: "21285" uid: 5059a9ad-6ccc-4be9-af46-de1ebfb5192d kind: Event lastTimestamp: "2026-07-01T18:27:14Z" message: 'Created pod: raw-sklearn-runtime-10c8f-predictor-649b8c799f-2hld6' metadata: creationTimestamp: "2026-07-01T18:27:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:27:14Z" name: raw-sklearn-runtime-10c8f-predictor-649b8c799f.18be3dc89d73b8c3 namespace: kserve-ci-e2e-test resourceVersion: "21298" uid: 8391a6b2-400a-4ebd-b0a6-19daf2615993 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-10c8f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21283" uid: 9cea13d4-6b69-4e8f-9b4f-788b787a319d kind: Event lastTimestamp: "2026-07-01T18:27:14Z" message: Scaled up replica set raw-sklearn-runtime-10c8f-predictor-649b8c799f from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:27:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:27:14Z" name: raw-sklearn-runtime-10c8f-predictor.18be3dc89889e84d namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: da4ade29-5631-4293-83cd-353971195e8f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:27:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-10c8f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21284" uid: 558825bc-3680-4e0c-a5fc-66d57892978f kind: Event lastTimestamp: "2026-07-01T18:27:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:27:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:27:44Z" name: raw-sklearn-runtime-10c8f-predictor.18be3dcc1720fb36 namespace: kserve-ci-e2e-test resourceVersion: "21548" uid: 15314383-0019-4eee-9ae1-90f22551e2c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:27:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-10c8f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21284" uid: 558825bc-3680-4e0c-a5fc-66d57892978f kind: Event lastTimestamp: "2026-07-01T18:27:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:27:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:27:44Z" name: raw-sklearn-runtime-10c8f-predictor.18be3dcc1782ebc0 namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: 755eb294-ecd2-42d6-97a5-d41819f8d1de reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:27:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-10c8f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: 558825bc-3680-4e0c-a5fc-66d57892978f kind: Event lastTimestamp: "2026-07-01T18:28:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:27:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:28:44Z" name: raw-sklearn-runtime-10c8f-predictor.18be3dd315a16854 namespace: kserve-ci-e2e-test resourceVersion: "21981" uid: 9d8bfb88-1f35-423b-8b20-b281dc5a2fcd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:27:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-10c8f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: 558825bc-3680-4e0c-a5fc-66d57892978f kind: Event lastTimestamp: "2026-07-01T18:28:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:27:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:28:44Z" name: raw-sklearn-runtime-10c8f-predictor.18be3dd31601c114 namespace: kserve-ci-e2e-test resourceVersion: "21982" uid: 8fd43d5b-1340-48ec-9eb4-0a982224d82b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T18:27:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-10c8f namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 76640576-64b6-40a9-8542-d26d0c4b9af3 kind: Event lastTimestamp: "2026-07-01T18:28:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:27:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:28:42Z" name: raw-sklearn-runtime-10c8f.18be3dc898a91560 namespace: kserve-ci-e2e-test resourceVersion: "21964" uid: 43eb69ce-27b8-4b8a-ba15-9666e6bfe5ff reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:27:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-10c8f namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 76640576-64b6-40a9-8542-d26d0c4b9af3 kind: Event lastTimestamp: "2026-07-01T18:27:14Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-10c8f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-10c8f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:27:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:27:14Z" name: raw-sklearn-runtime-10c8f.18be3dc8992ab26e namespace: kserve-ci-e2e-test resourceVersion: "21291" uid: 6ea3875d-7244-48bf-84c2-1d8e518129fc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:28:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-10c8f namespace: kserve-ci-e2e-test resourceVersion: "21962" uid: 76640576-64b6-40a9-8542-d26d0c4b9af3 kind: Event lastTimestamp: "2026-07-01T18:28:41Z" message: InferenceService [raw-sklearn-runtime-10c8f] is Ready metadata: creationTimestamp: "2026-07-01T18:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:28:41Z" name: raw-sklearn-runtime-10c8f.18be3ddd04f41621 namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: ad6cb27b-ae9c-4a50-a05a-e6d37dcedf7b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22251"