--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:47:45.988180Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-d6e639-predictor-5449f794b-jlrqj namespace: kserve-ci-e2e-test resourceVersion: "20802" uid: f1363ccb-7c98-4c7f-bf24-4052e27489eb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-d6e639-predictor-5449f794b-jlrqj to ip-10-0-136-31.ec2.internal metadata: creationTimestamp: "2026-07-21T17:47:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:47:45Z" name: isvc-init-fail-d6e639-predictor-5449f794b-jlrqj.18c45f3c8b78d571 namespace: kserve-ci-e2e-test resourceVersion: "20808" uid: f36cc4f2-e189-4726-a025-ab8d6d16305e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf7f8bd85-gq8p9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:46Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-d6e639-predictor-5449f794b-jlrqj namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: f1363ccb-7c98-4c7f-bf24-4052e27489eb kind: Event lastTimestamp: "2026-07-21T17:47:46Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:47:46Z" name: isvc-init-fail-d6e639-predictor-5449f794b-jlrqj.18c45f3ca5fe5e5f namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 9a74957f-77f3-42cc-b993-ad7be89ff1dc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:47:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-d6e639-predictor-5449f794b-jlrqj namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: f1363ccb-7c98-4c7f-bf24-4052e27489eb kind: Event lastTimestamp: "2026-07-21T17:47:51Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e12dde312cd424230e1ef8053d8890ee8a23543039767c0a577fe647c927b0a" already present on machine metadata: creationTimestamp: "2026-07-21T17:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:51Z" name: isvc-init-fail-d6e639-predictor-5449f794b-jlrqj.18c45f3ca7883429 namespace: kserve-ci-e2e-test resourceVersion: "20866" uid: 129e319a-0524-456d-b06f-455cac035bd2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:47:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-d6e639-predictor-5449f794b-jlrqj namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: f1363ccb-7c98-4c7f-bf24-4052e27489eb kind: Event lastTimestamp: "2026-07-21T17:47:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-21T17:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:51Z" name: isvc-init-fail-d6e639-predictor-5449f794b-jlrqj.18c45f3caa9f0937 namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: 0a4f73c2-8dfb-4573-b90f-f532465b93ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:47:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-d6e639-predictor-5449f794b-jlrqj namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: f1363ccb-7c98-4c7f-bf24-4052e27489eb kind: Event lastTimestamp: "2026-07-21T17:47:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-21T17:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:51Z" name: isvc-init-fail-d6e639-predictor-5449f794b-jlrqj.18c45f3cab0e0835 namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: 2fda4344-de54-445b-bf2c-c3f182b6b06d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-d6e639-predictor-5449f794b-jlrqj namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: f1363ccb-7c98-4c7f-bf24-4052e27489eb kind: Event lastTimestamp: "2026-07-21T17:47:54Z" message: Back-off restarting failed container storage-initializer in pod isvc-init-fail-d6e639-predictor-5449f794b-jlrqj_kserve-ci-e2e-test(f1363ccb-7c98-4c7f-bf24-4052e27489eb) metadata: creationTimestamp: "2026-07-21T17:47:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:54Z" name: isvc-init-fail-d6e639-predictor-5449f794b-jlrqj.18c45f3e90143fa8 namespace: kserve-ci-e2e-test resourceVersion: "20895" uid: db543abb-c493-4370-9ff0-ec07e906d282 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-d6e639-predictor-5449f794b namespace: kserve-ci-e2e-test resourceVersion: "20793" uid: b29ddbd4-179c-4f93-ad09-9f1b1f9da05c kind: Event lastTimestamp: "2026-07-21T17:47:45Z" message: 'Created pod: isvc-init-fail-d6e639-predictor-5449f794b-jlrqj' metadata: creationTimestamp: "2026-07-21T17:47:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:47:45Z" name: isvc-init-fail-d6e639-predictor-5449f794b.18c45f3c8aed05c8 namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: e05108df-769f-47c5-9254-7e2766cf046c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-d6e639-predictor namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: 27264e88-a30d-4b6e-944c-b7685508657e kind: Event lastTimestamp: "2026-07-21T17:47:45Z" message: Scaled up replica set isvc-init-fail-d6e639-predictor-5449f794b from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:47:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:47:45Z" name: isvc-init-fail-d6e639-predictor.18c45f3c882aa38d namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 982a90e2-edb8-4fdf-bcac-ff0003ad8a07 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-21T17:47:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-d6e639 namespace: kserve-ci-e2e-test resourceVersion: "20784" uid: 747cb3a0-d541-4e55-8797-1b69b16eafa2 kind: Event lastTimestamp: "2026-07-21T17:47:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:47:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:47:54Z" name: isvc-init-fail-d6e639.18c45f3c8850e4ce namespace: kserve-ci-e2e-test resourceVersion: "20901" uid: 12b1c01f-e564-45a4-903b-80222612e4cc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-d6e639 namespace: kserve-ci-e2e-test resourceVersion: "20784" uid: 747cb3a0-d541-4e55-8797-1b69b16eafa2 kind: Event lastTimestamp: "2026-07-21T17:47:45Z" message: 'Failed to update status for InferenceService "isvc-init-fail-d6e639": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-d6e639": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:47:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:47:45Z" name: isvc-init-fail-d6e639.18c45f3c88c851c1 namespace: kserve-ci-e2e-test resourceVersion: "20797" uid: e95e6a2a-3028-40f1-913d-3159e57ae49d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-d6e639 namespace: kserve-ci-e2e-test resourceVersion: "20784" uid: 747cb3a0-d541-4e55-8797-1b69b16eafa2 kind: Event lastTimestamp: "2026-07-21T17:47:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-d6e639": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:47:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:47:45Z" name: isvc-init-fail-d6e639.18c45f3c88c89554 namespace: kserve-ci-e2e-test resourceVersion: "20798" uid: 0a6fa446-b07b-4076-b5ed-50a3832a6abb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:41:18.866347Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 namespace: kserve-ci-e2e-test resourceVersion: "17676" uid: 45b45d18-c05d-420d-9ef8-4da31c1fc48f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 to ip-10-0-136-31.ec2.internal metadata: creationTimestamp: "2026-07-21T17:41:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:41:18Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5.18c45ee26936758c namespace: kserve-ci-e2e-test resourceVersion: "17682" uid: edbb7a10-0061-4ed8-82f5-5d949b9d7ca1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf7f8bd85-gq8p9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:19Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 namespace: kserve-ci-e2e-test resourceVersion: "17687" uid: 45b45d18-c05d-420d-9ef8-4da31c1fc48f kind: Event lastTimestamp: "2026-07-21T17:41:19Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:41:19Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5.18c45ee28303d78a namespace: kserve-ci-e2e-test resourceVersion: "17694" uid: ea1324f9-2b31-4b44-bdbf-586b1a8b2036 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 45b45d18-c05d-420d-9ef8-4da31c1fc48f kind: Event lastTimestamp: "2026-07-21T17:41:19Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e12dde312cd424230e1ef8053d8890ee8a23543039767c0a577fe647c927b0a" already present on machine metadata: creationTimestamp: "2026-07-21T17:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:19Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5.18c45ee284b1c8ab namespace: kserve-ci-e2e-test resourceVersion: "17697" uid: 952c1c75-9e62-46ed-95d2-f16328f96359 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 45b45d18-c05d-420d-9ef8-4da31c1fc48f kind: Event lastTimestamp: "2026-07-21T17:41:19Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-21T17:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:19Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5.18c45ee2877eac1f namespace: kserve-ci-e2e-test resourceVersion: "17698" uid: ca41aadf-6ea8-481f-b433-1a7f0cf95cbd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 45b45d18-c05d-420d-9ef8-4da31c1fc48f kind: Event lastTimestamp: "2026-07-21T17:41:19Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-21T17:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:19Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5.18c45ee287ec44e8 namespace: kserve-ci-e2e-test resourceVersion: "17699" uid: 200a6ff9-ba5b-4bc8-b2ab-4dd4ca4fd7c1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 45b45d18-c05d-420d-9ef8-4da31c1fc48f kind: Event lastTimestamp: "2026-07-21T17:41:23Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:41:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:23Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5.18c45ee39638b32c namespace: kserve-ci-e2e-test resourceVersion: "17738" uid: 48406754-51d2-4411-b646-691395c239de reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 45b45d18-c05d-420d-9ef8-4da31c1fc48f kind: Event lastTimestamp: "2026-07-21T17:41:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:41:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:23Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5.18c45ee3997a17d7 namespace: kserve-ci-e2e-test resourceVersion: "17742" uid: f0378bff-28fa-4718-a6dc-113b81e82844 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 45b45d18-c05d-420d-9ef8-4da31c1fc48f kind: Event lastTimestamp: "2026-07-21T17:41:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:41:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:23Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5.18c45ee39a0d021f namespace: kserve-ci-e2e-test resourceVersion: "17743" uid: 1357d53a-5052-4943-912a-216fbbe4a17d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 45b45d18-c05d-420d-9ef8-4da31c1fc48f kind: Event lastTimestamp: "2026-07-21T17:41:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:41:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:23Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5.18c45ee39a17398e namespace: kserve-ci-e2e-test resourceVersion: "17744" uid: 579ba6f5-0609-484f-8e98-72fadeac32dc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 45b45d18-c05d-420d-9ef8-4da31c1fc48f kind: Event lastTimestamp: "2026-07-21T17:41:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:41:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:24Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5.18c45ee39d2427fc namespace: kserve-ci-e2e-test resourceVersion: "17747" uid: b17820b7-f01a-41c2-ad58-f8d8a9bec932 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 45b45d18-c05d-420d-9ef8-4da31c1fc48f kind: Event lastTimestamp: "2026-07-21T17:41:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:41:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:24Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5.18c45ee39db6f468 namespace: kserve-ci-e2e-test resourceVersion: "17748" uid: 128b6de4-b1a3-402a-adf0-1e18702492e7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 45b45d18-c05d-420d-9ef8-4da31c1fc48f kind: Event lastTimestamp: "2026-07-21T17:41:24Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:a6dd057fcbb6a3a050885e9fca53c6c0863f884934b195f20c7d1ba14dfd564c" already present on machine metadata: creationTimestamp: "2026-07-21T17:41:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:24Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5.18c45ee39dc10671 namespace: kserve-ci-e2e-test resourceVersion: "17749" uid: 0753860b-a213-4e47-b200-5c8cf0686f3c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 45b45d18-c05d-420d-9ef8-4da31c1fc48f kind: Event lastTimestamp: "2026-07-21T17:41:24Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-21T17:41:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:24Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5.18c45ee3a0f5e06a namespace: kserve-ci-e2e-test resourceVersion: "17751" uid: 39500630-4f8f-432a-ac5f-7c9a9386aafe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 45b45d18-c05d-420d-9ef8-4da31c1fc48f kind: Event lastTimestamp: "2026-07-21T17:41:24Z" message: Started container agent metadata: creationTimestamp: "2026-07-21T17:41:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:24Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5.18c45ee3a168003b namespace: kserve-ci-e2e-test resourceVersion: "17752" uid: 06e1bc03-1b8c-4a9d-98c1-a15df89d50bd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-21T17:41:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 45b45d18-c05d-420d-9ef8-4da31c1fc48f kind: Event lastTimestamp: "2026-07-21T17:43:11Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:41:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:43:11Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5.18c45ee40e32bb6f namespace: kserve-ci-e2e-test resourceVersion: "18647" uid: f07f5f57-d7c7-4e0d-84d9-ff9a5166380f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-21T17:41:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 45b45d18-c05d-420d-9ef8-4da31c1fc48f kind: Event lastTimestamp: "2026-07-21T17:43:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-21T17:41:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:43:01Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5.18c45ee40e3d5465 namespace: kserve-ci-e2e-test resourceVersion: "18571" uid: 18616f95-2822-45b4-9cc7-99e499b68525 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:42:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 45b45d18-c05d-420d-9ef8-4da31c1fc48f kind: Event lastTimestamp: "2026-07-21T17:42:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:42:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:42:54Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5.18c45ef896201ba9 namespace: kserve-ci-e2e-test resourceVersion: "18440" uid: 1de29ff5-8c01-4a32-9b90-a290f63a4e21 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:42:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 45b45d18-c05d-420d-9ef8-4da31c1fc48f kind: Event lastTimestamp: "2026-07-21T17:42:54Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-21T17:42:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:42:54Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5.18c45ef89620621a namespace: kserve-ci-e2e-test resourceVersion: "18443" uid: d4348847-0af9-4fc2-83d0-1c9348f7170e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:42:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 45b45d18-c05d-420d-9ef8-4da31c1fc48f kind: Event lastTimestamp: "2026-07-21T17:42:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:42:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:42:54Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5.18c45ef89620f6d4 namespace: kserve-ci-e2e-test resourceVersion: "18446" uid: 2cc2273e-90be-41df-b59e-532bcbfd96a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-21T17:42:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 45b45d18-c05d-420d-9ef8-4da31c1fc48f kind: Event lastTimestamp: "2026-07-21T17:43:11Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:42:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:43:11Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5.18c45ef93e704fc5 namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: be0a3318-7637-4540-8dc8-55c06cc795e2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-4098a-predictor-7d7bdfd689 namespace: kserve-ci-e2e-test resourceVersion: "17667" uid: 3177ade4-1525-4073-bd46-2f73c3ec05ab kind: Event lastTimestamp: "2026-07-21T17:41:18Z" message: 'Created pod: isvc-logger-raw-4098a-predictor-7d7bdfd689-288x5' metadata: creationTimestamp: "2026-07-21T17:41:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:41:18Z" name: isvc-logger-raw-4098a-predictor-7d7bdfd689.18c45ee268ce6221 namespace: kserve-ci-e2e-test resourceVersion: "17679" uid: 188901cc-a1c7-46d8-99ac-dfee6956cc48 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-4098a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17665" uid: 88a7f753-aaa7-4b2b-ac38-73f217303ac5 kind: Event lastTimestamp: "2026-07-21T17:41:18Z" message: Scaled up replica set isvc-logger-raw-4098a-predictor-7d7bdfd689 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:41:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:41:18Z" name: isvc-logger-raw-4098a-predictor.18c45ee265582663 namespace: kserve-ci-e2e-test resourceVersion: "17668" uid: ab5afbf6-87ca-48e5-a681-36fca513a569 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-4098a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17666" uid: 4cf22709-f7a1-4748-ab98-013256425fa2 kind: Event lastTimestamp: "2026-07-21T17:41:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:41:33Z" name: isvc-logger-raw-4098a-predictor.18c45ee5e408cc35 namespace: kserve-ci-e2e-test resourceVersion: "17831" uid: f3107cc4-56cb-4782-bac1-1b5fbbefded8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-4098a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17666" uid: 4cf22709-f7a1-4748-ab98-013256425fa2 kind: Event lastTimestamp: "2026-07-21T17:41:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:41:33Z" name: isvc-logger-raw-4098a-predictor.18c45ee5e46c4b04 namespace: kserve-ci-e2e-test resourceVersion: "17832" uid: 5e3e1a5f-2707-446b-8081-bf6c89d97e74 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-21T17:41:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-4098a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17830" uid: 4cf22709-f7a1-4748-ab98-013256425fa2 kind: Event lastTimestamp: "2026-07-21T17:42:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:42:48Z" name: isvc-logger-raw-4098a-predictor.18c45ee9633d7b6b namespace: kserve-ci-e2e-test resourceVersion: "18369" uid: 8d1d1b7a-73d6-44f6-a2ad-ff3e6819ea96 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-21T17:41:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-4098a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17830" uid: 4cf22709-f7a1-4748-ab98-013256425fa2 kind: Event lastTimestamp: "2026-07-21T17:42:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:42:48Z" name: isvc-logger-raw-4098a-predictor.18c45ee963b47993 namespace: kserve-ci-e2e-test resourceVersion: "18370" uid: ee70822f-02a0-4fbd-b162-c301a0750291 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-21T17:41:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-4098a namespace: kserve-ci-e2e-test resourceVersion: "17657" uid: 3b52bfd8-bb57-4a7f-954b-b10322f5d790 kind: Event lastTimestamp: "2026-07-21T17:42:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:41:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:42:42Z" name: isvc-logger-raw-4098a.18c45ee26565c0fc namespace: kserve-ci-e2e-test resourceVersion: "18326" uid: 5fa6db7e-dcf2-4591-841f-a0e8f9442b16 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-4098a namespace: kserve-ci-e2e-test resourceVersion: "17657" uid: 3b52bfd8-bb57-4a7f-954b-b10322f5d790 kind: Event lastTimestamp: "2026-07-21T17:41:18Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-4098a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-4098a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:41:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:41:18Z" name: isvc-logger-raw-4098a.18c45ee265f38815 namespace: kserve-ci-e2e-test resourceVersion: "17671" uid: 3811adb2-fc96-4dc3-a5fd-3e012c943c2a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:42:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-4098a namespace: kserve-ci-e2e-test resourceVersion: "18324" uid: 3b52bfd8-bb57-4a7f-954b-b10322f5d790 kind: Event lastTimestamp: "2026-07-21T17:42:42Z" message: InferenceService [isvc-logger-raw-4098a] is Ready metadata: creationTimestamp: "2026-07-21T17:42:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:42:42Z" name: isvc-logger-raw-4098a.18c45ef5c7d3cfc1 namespace: kserve-ci-e2e-test resourceVersion: "18325" uid: 9a239400-5152-403f-902d-21810a29391f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:45:39.499336Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5 namespace: kserve-ci-e2e-test resourceVersion: "19732" uid: ca915421-af3e-4be7-80d4-3ef5f19bdcdf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5 to ip-10-0-136-31.ec2.internal metadata: creationTimestamp: "2026-07-21T17:45:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:45:39Z" name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5.18c45f1f18264a4e namespace: kserve-ci-e2e-test resourceVersion: "19736" uid: 5413b044-f0b8-4f59-abb1-518a90c712fb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf7f8bd85-gq8p9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:45:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5 namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: ca915421-af3e-4be7-80d4-3ef5f19bdcdf kind: Event lastTimestamp: "2026-07-21T17:45:39Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:45:39Z" name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5.18c45f1f3269e183 namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 6c4c8f24-e632-4dc9-b258-b24c799ea1ce reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5 namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: ca915421-af3e-4be7-80d4-3ef5f19bdcdf kind: Event lastTimestamp: "2026-07-21T17:45:39Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e12dde312cd424230e1ef8053d8890ee8a23543039767c0a577fe647c927b0a" already present on machine metadata: creationTimestamp: "2026-07-21T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:45:39Z" name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5.18c45f1f33dbe20f namespace: kserve-ci-e2e-test resourceVersion: "19754" uid: 9c834e61-5f36-49c6-abae-435135440c5b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:45:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5 namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: ca915421-af3e-4be7-80d4-3ef5f19bdcdf kind: Event lastTimestamp: "2026-07-21T17:45:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-21T17:45:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:45:40Z" name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5.18c45f1f3698d768 namespace: kserve-ci-e2e-test resourceVersion: "19756" uid: b14f8190-5b34-46e3-ba8a-58297ce7be0a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:45:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5 namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: ca915421-af3e-4be7-80d4-3ef5f19bdcdf kind: Event lastTimestamp: "2026-07-21T17:45:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-21T17:45:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:45:40Z" name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5.18c45f1f370f7332 namespace: kserve-ci-e2e-test resourceVersion: "19757" uid: b376092b-649c-4a96-afaa-8e42b2a711c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:45:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5 namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: ca915421-af3e-4be7-80d4-3ef5f19bdcdf kind: Event lastTimestamp: "2026-07-21T17:45:44Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:45:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:45:44Z" name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5.18c45f202811c6d6 namespace: kserve-ci-e2e-test resourceVersion: "19793" uid: 2b7c1c46-1452-44ce-98f1-83b154e67187 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:45:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5 namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: ca915421-af3e-4be7-80d4-3ef5f19bdcdf kind: Event lastTimestamp: "2026-07-21T17:45:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:45:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:45:44Z" name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5.18c45f202d3add13 namespace: kserve-ci-e2e-test resourceVersion: "19797" uid: 6499ab28-f9fa-492a-acac-b09278be0e3b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:45:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5 namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: ca915421-af3e-4be7-80d4-3ef5f19bdcdf kind: Event lastTimestamp: "2026-07-21T17:45:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:45:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:45:44Z" name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5.18c45f202db25c59 namespace: kserve-ci-e2e-test resourceVersion: "19798" uid: b9b559f5-0004-4bcc-be87-14cd163e0416 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:45:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5 namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: ca915421-af3e-4be7-80d4-3ef5f19bdcdf kind: Event lastTimestamp: "2026-07-21T17:45:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:45:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:45:44Z" name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5.18c45f202dbc8e0c namespace: kserve-ci-e2e-test resourceVersion: "19799" uid: 185e52d7-5e14-44a8-a45b-f6999f956f8d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:45:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5 namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: ca915421-af3e-4be7-80d4-3ef5f19bdcdf kind: Event lastTimestamp: "2026-07-21T17:45:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:45:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:45:44Z" name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5.18c45f20327b6de4 namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: 455985eb-88ec-4c5f-9a1c-2f080b4d64de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:45:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5 namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: ca915421-af3e-4be7-80d4-3ef5f19bdcdf kind: Event lastTimestamp: "2026-07-21T17:45:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:45:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:45:44Z" name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5.18c45f2032f2921d namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: c94e9f1a-63b9-4bd8-adbb-d3a73ec56453 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-21T17:45:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5 namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: ca915421-af3e-4be7-80d4-3ef5f19bdcdf kind: Event lastTimestamp: "2026-07-21T17:47:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:45:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:41Z" name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5.18c45f20649cfd7d namespace: kserve-ci-e2e-test resourceVersion: "20735" uid: cd52d2de-7f2c-4717-8970-4837bdf0c6ae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5 namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: ca915421-af3e-4be7-80d4-3ef5f19bdcdf kind: Event lastTimestamp: "2026-07-21T17:47:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:40Z" name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5.18c45f3b57df5596 namespace: kserve-ci-e2e-test resourceVersion: "20728" uid: 1e9a7af2-14e7-4d85-bc95-c9a07fd2b1f8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5 namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: ca915421-af3e-4be7-80d4-3ef5f19bdcdf kind: Event lastTimestamp: "2026-07-21T17:47:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:40Z" name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5.18c45f3b57dffba3 namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: 0e411cc2-46ca-4c51-bdda-057c90502004 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5 namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: ca915421-af3e-4be7-80d4-3ef5f19bdcdf kind: Event lastTimestamp: "2026-07-21T17:47:41Z" message: 'Readiness probe failed: Get "https://10.133.0.38:8643/healthz": dial tcp 10.133.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:41Z" name: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5.18c45f3b670c3f77 namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: 1f681fdc-595a-40f0-8b94-5447ef624c39 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:45:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-4e9225-predictor-65bcb5bcfc namespace: kserve-ci-e2e-test resourceVersion: "19723" uid: ab3c16d7-9bf9-478c-865d-d62ed85f3a4f kind: Event lastTimestamp: "2026-07-21T17:45:39Z" message: 'Created pod: isvc-primary-4e9225-predictor-65bcb5bcfc-ngrn5' metadata: creationTimestamp: "2026-07-21T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:45:39Z" name: isvc-primary-4e9225-predictor-65bcb5bcfc.18c45f1f17b810d1 namespace: kserve-ci-e2e-test resourceVersion: "19733" uid: c1642ec0-3412-439e-8dce-9e43c64d3a30 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:45:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-4e9225-predictor namespace: kserve-ci-e2e-test resourceVersion: "19721" uid: a89c1610-3274-4ccd-8365-f40ceff1cf51 kind: Event lastTimestamp: "2026-07-21T17:45:39Z" message: Scaled up replica set isvc-primary-4e9225-predictor-65bcb5bcfc from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:45:39Z" name: isvc-primary-4e9225-predictor.18c45f1f135f8811 namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 86381082-f24e-4598-a03c-00f79eaf9a94 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:45:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-4e9225-predictor namespace: kserve-ci-e2e-test resourceVersion: "19722" uid: 3d974c3c-4352-4b81-b7ec-f5f5a9081f64 kind: Event lastTimestamp: "2026-07-21T17:46:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:45:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:46:24Z" name: isvc-primary-4e9225-predictor.18c45f2291e39af5 namespace: kserve-ci-e2e-test resourceVersion: "20087" uid: e3b7c8b7-e9ad-41ac-9a45-b7a1c27c32c1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:45:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-4e9225-predictor namespace: kserve-ci-e2e-test resourceVersion: "19722" uid: 3d974c3c-4352-4b81-b7ec-f5f5a9081f64 kind: Event lastTimestamp: "2026-07-21T17:46:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:45:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:46:24Z" name: isvc-primary-4e9225-predictor.18c45f22924ffeda namespace: kserve-ci-e2e-test resourceVersion: "20088" uid: e3360035-f41f-4ce8-a62a-28683db8f2e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:46:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-4e9225-predictor namespace: kserve-ci-e2e-test resourceVersion: "19875" uid: 3d974c3c-4352-4b81-b7ec-f5f5a9081f64 kind: Event lastTimestamp: "2026-07-21T17:47:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:46:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:47:39Z" name: isvc-primary-4e9225-predictor.18c45f2611aa95c8 namespace: kserve-ci-e2e-test resourceVersion: "20683" uid: 194b8b35-5a32-4ae1-83b4-403624be566c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:46:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-4e9225-predictor namespace: kserve-ci-e2e-test resourceVersion: "19875" uid: 3d974c3c-4352-4b81-b7ec-f5f5a9081f64 kind: Event lastTimestamp: "2026-07-21T17:47:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:46:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:47:39Z" name: isvc-primary-4e9225-predictor.18c45f26122f30e6 namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: b6517be6-fdef-4604-8fbf-cce472c83639 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-21T17:45:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-4e9225 namespace: kserve-ci-e2e-test resourceVersion: "19714" uid: e6ee89e1-ba5f-4b6c-9d79-310f44f89ec2 kind: Event lastTimestamp: "2026-07-21T17:47:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:47:01Z" name: isvc-primary-4e9225.18c45f1f135f4ba8 namespace: kserve-ci-e2e-test resourceVersion: "20354" uid: 1166b494-5488-4577-9486-7081ab26398b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:45:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-4e9225 namespace: kserve-ci-e2e-test resourceVersion: "19714" uid: e6ee89e1-ba5f-4b6c-9d79-310f44f89ec2 kind: Event lastTimestamp: "2026-07-21T17:45:39Z" message: 'Failed to update status for InferenceService "isvc-primary-4e9225": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-4e9225": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:45:39Z" name: isvc-primary-4e9225.18c45f1f140669a1 namespace: kserve-ci-e2e-test resourceVersion: "19727" uid: a96ffa8a-82b2-417c-9e79-914693409716 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:45:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-4e9225 namespace: kserve-ci-e2e-test resourceVersion: "19714" uid: e6ee89e1-ba5f-4b6c-9d79-310f44f89ec2 kind: Event lastTimestamp: "2026-07-21T17:45:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-4e9225": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:45:39Z" name: isvc-primary-4e9225.18c45f1f1406a861 namespace: kserve-ci-e2e-test resourceVersion: "19728" uid: 5abc7211-6fed-4faf-9f1c-d8b9c71b4b38 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-4e9225 namespace: kserve-ci-e2e-test resourceVersion: "20352" uid: e6ee89e1-ba5f-4b6c-9d79-310f44f89ec2 kind: Event lastTimestamp: "2026-07-21T17:47:01Z" message: InferenceService [isvc-primary-4e9225] is Ready metadata: creationTimestamp: "2026-07-21T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:47:01Z" name: isvc-primary-4e9225.18c45f321c595c1f namespace: kserve-ci-e2e-test resourceVersion: "20353" uid: 1b0dd588-21fc-431b-a887-2eb27894a9b0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:35:23.046328Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14447" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz to ip-10-0-136-31.ec2.internal metadata: creationTimestamp: "2026-07-21T17:35:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:35:23Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45e8f90afdadb namespace: kserve-ci-e2e-test resourceVersion: "14451" uid: a9d38043-e135-4954-9ad2-b3bc3ef71a46 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf7f8bd85-gq8p9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:23Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:35:23Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:35:23Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45e8fa9fb448a namespace: kserve-ci-e2e-test resourceVersion: "14470" uid: 5285b879-d656-4acc-b443-197ba638312f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:35:23Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e12dde312cd424230e1ef8053d8890ee8a23543039767c0a577fe647c927b0a" metadata: creationTimestamp: "2026-07-21T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:23Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45e8fab8c4fd0 namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: ece7dfba-d6c1-4994-9399-f37443a13433 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:35:26Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e12dde312cd424230e1ef8053d8890ee8a23543039767c0a577fe647c927b0a" in 3.4s (3.4s including waiting). Image size: 301914034 bytes.' metadata: creationTimestamp: "2026-07-21T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:26Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45e907639cec1 namespace: kserve-ci-e2e-test resourceVersion: "14496" uid: f92416d2-60a0-4c70-8be5-22c1a1cb39ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:35:26Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-21T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:26Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45e90795bda21 namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 0de73169-9fae-4f0e-a1d8-8ac4f8a4bda6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:35:26Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-21T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:26Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45e9079cb73e8 namespace: kserve-ci-e2e-test resourceVersion: "14498" uid: 1558223c-a3e2-4468-9442-19aa40a05345 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:35:31Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1775" metadata: creationTimestamp: "2026-07-21T17:35:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:31Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45e917ec3ebf3 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: c5d5a68d-fd73-47f6-8767-5c786b92a2f6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:35:45Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1775" in 13.983s (13.983s including waiting). Image size: 1558090149 bytes.' metadata: creationTimestamp: "2026-07-21T17:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:45Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45e94c03eb215 namespace: kserve-ci-e2e-test resourceVersion: "14642" uid: 0625c73c-ebdd-4596-a879-ed8dc842800b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:35:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:45Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45e94c33c2a71 namespace: kserve-ci-e2e-test resourceVersion: "14643" uid: c73626e3-35be-4e68-9f2a-1c14d874c808 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:35:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:45Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45e94c3adcd20 namespace: kserve-ci-e2e-test resourceVersion: "14644" uid: 89d2c0f4-2f6a-4444-b78e-e8901cc8dfdd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:35:45Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-21T17:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:45Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45e94c3ba9cc1 namespace: kserve-ci-e2e-test resourceVersion: "14645" uid: b92bd1d4-fbb4-44c7-968a-a3629f23b6b5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:35:47Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.308s (2.308s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-21T17:35:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:47Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45e954d534a39 namespace: kserve-ci-e2e-test resourceVersion: "14661" uid: 931a74fc-2758-420c-80bd-09ceabe908ea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:35:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:35:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:47Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45e9550bd4797 namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 40fada3a-763f-4a32-a844-dda52d43039a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:35:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:35:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:47Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45e95512f396e namespace: kserve-ci-e2e-test resourceVersion: "14663" uid: e507d7da-9e01-4ade-b778-0ca357a96480 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:35:47Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:a6dd057fcbb6a3a050885e9fca53c6c0863f884934b195f20c7d1ba14dfd564c" metadata: creationTimestamp: "2026-07-21T17:35:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:47Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45e95513ae6b7 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: bf19c408-ac3f-43b4-96a8-6361eaa1dfcd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:35:50Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:a6dd057fcbb6a3a050885e9fca53c6c0863f884934b195f20c7d1ba14dfd564c" in 2.964s (2.964s including waiting). Image size: 240172118 bytes.' metadata: creationTimestamp: "2026-07-21T17:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:50Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45e9601f1dc1c namespace: kserve-ci-e2e-test resourceVersion: "14681" uid: ff39e61b-6296-4f85-a693-4c961dbea1f8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:35:50Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-21T17:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:50Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45e96071a2d15 namespace: kserve-ci-e2e-test resourceVersion: "14683" uid: 3fa2150b-d66a-4776-8ab1-1491cfd070a6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:35:50Z" message: Started container agent metadata: creationTimestamp: "2026-07-21T17:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:50Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45e96080fdfc4 namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: b3c1a055-3dfb-4c24-bd0d-a172859d5334 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-21T17:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:37:34Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:34Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45e962d00f210 namespace: kserve-ci-e2e-test resourceVersion: "15585" uid: af86a75f-ca3d-417e-858a-f39d17fd8e13 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-21T17:35:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:37:24Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-21T17:35:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:24Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45e9668d0db5b namespace: kserve-ci-e2e-test resourceVersion: "15506" uid: a7ad678a-4f35-40c2-a68d-e471976e8bd4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:37:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:18Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45eaa59401c49 namespace: kserve-ci-e2e-test resourceVersion: "15359" uid: 10f1fd05-3a92-4da9-9e4a-0fb789c0ee85 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:37:18Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:18Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45eaa59407167 namespace: kserve-ci-e2e-test resourceVersion: "15363" uid: ee4c00c0-d31b-4703-8e45-f60cab54252b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:37:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:18Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45eaa59434192 namespace: kserve-ci-e2e-test resourceVersion: "15365" uid: b0224024-2700-4650-8121-c327891deeff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:37:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 729f852e-7f2e-4037-bfc6-bb8c1b8d9001 kind: Event lastTimestamp: "2026-07-21T17:37:32Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:32Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz.18c45eab5d2e8cbd namespace: kserve-ci-e2e-test resourceVersion: "15565" uid: f17c1474-8519-44e7-aa4b-6795eea19454 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584 namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: d5974787-4d4e-4f76-9920-e18ed226d2e8 kind: Event lastTimestamp: "2026-07-21T17:35:23Z" message: 'Created pod: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584-xc7rz' metadata: creationTimestamp: "2026-07-21T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:35:23Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584.18c45e8f903d085e namespace: kserve-ci-e2e-test resourceVersion: "14448" uid: a58a45ff-ab6c-4c6e-9f98-133330212009 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-e8d4c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 70a12ff9-a57a-43d8-8195-1802cea0f8a2 kind: Event lastTimestamp: "2026-07-21T17:35:22Z" message: Scaled up replica set isvc-raw-sklearn-batcher-e8d4c-predictor-6c5c65c584 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:35:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:35:22Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor.18c45e8f8b43faf4 namespace: kserve-ci-e2e-test resourceVersion: "14442" uid: c3e3273e-a36d-4fb1-a310-4abd4ba34ec6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:35:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-e8d4c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14452" uid: 78a7f62b-c012-4491-87bf-db2369d185f8 kind: Event lastTimestamp: "2026-07-21T17:36:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:36:23Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor.18c45e9311d977cd namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: d87d4a99-3705-4436-8d59-5e1567687a40 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:35:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-e8d4c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14452" uid: 78a7f62b-c012-4491-87bf-db2369d185f8 kind: Event lastTimestamp: "2026-07-21T17:36:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:36:23Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor.18c45e931254f456 namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: 2cfce508-4833-4180-a78e-114bb1d6ee2d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-21T17:36:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-e8d4c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: 78a7f62b-c012-4491-87bf-db2369d185f8 kind: Event lastTimestamp: "2026-07-21T17:37:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:36:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:08Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor.18c45e9a10afa5e0 namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: a34dfb02-cd81-407e-a172-956b2d7e518a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-21T17:36:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-e8d4c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: 78a7f62b-c012-4491-87bf-db2369d185f8 kind: Event lastTimestamp: "2026-07-21T17:37:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:36:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:08Z" name: isvc-raw-sklearn-batcher-e8d4c-predictor.18c45e9a112627ce namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: af594c69-4097-4020-a018-cc1801390d79 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-21T17:35:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-e8d4c namespace: kserve-ci-e2e-test resourceVersion: "14433" uid: a4a391af-ad01-439d-9346-db966f7a8ed4 kind: Event lastTimestamp: "2026-07-21T17:37:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:37:04Z" name: isvc-raw-sklearn-batcher-e8d4c.18c45e8f9165146b namespace: kserve-ci-e2e-test resourceVersion: "15236" uid: 467369dc-8820-4531-8cd6-af4d920159ca reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-e8d4c namespace: kserve-ci-e2e-test resourceVersion: "14433" uid: a4a391af-ad01-439d-9346-db966f7a8ed4 kind: Event lastTimestamp: "2026-07-21T17:35:23Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-e8d4c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-e8d4c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:35:23Z" name: isvc-raw-sklearn-batcher-e8d4c.18c45e8f91d78832 namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: 5688e9d2-75f0-4c3f-8cc5-9a84f7e591c3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-e8d4c namespace: kserve-ci-e2e-test resourceVersion: "14433" uid: a4a391af-ad01-439d-9346-db966f7a8ed4 kind: Event lastTimestamp: "2026-07-21T17:35:23Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-e8d4c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:35:23Z" name: isvc-raw-sklearn-batcher-e8d4c.18c45e8f91d7f08c namespace: kserve-ci-e2e-test resourceVersion: "14458" uid: f06652d9-cc90-4e2a-b988-bfbc7ad6b8e8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-e8d4c namespace: kserve-ci-e2e-test resourceVersion: "15234" uid: a4a391af-ad01-439d-9346-db966f7a8ed4 kind: Event lastTimestamp: "2026-07-21T17:37:04Z" message: InferenceService [isvc-raw-sklearn-batcher-e8d4c] is Ready metadata: creationTimestamp: "2026-07-21T17:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:37:04Z" name: isvc-raw-sklearn-batcher-e8d4c.18c45ea73381e8b1 namespace: kserve-ci-e2e-test resourceVersion: "15235" uid: e7f66626-31f7-4824-a3a1-32df4b070c1c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:47:24.712874Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-4e9225-predictor-7c68b4f8-5nwt5 namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: 07b0984c-5c5f-453d-8f6a-018d0e6e3d02 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-4e9225-predictor-7c68b4f8-5nwt5 to ip-10-0-136-31.ec2.internal metadata: creationTimestamp: "2026-07-21T17:47:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:47:24Z" name: isvc-secondary-4e9225-predictor-7c68b4f8-5nwt5.18c45f37975d7702 namespace: kserve-ci-e2e-test resourceVersion: "20550" uid: d4bed137-ba31-45e9-b9da-bfe2bdf8e2ca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf7f8bd85-gq8p9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:25Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-4e9225-predictor-7c68b4f8-5nwt5 namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: 07b0984c-5c5f-453d-8f6a-018d0e6e3d02 kind: Event lastTimestamp: "2026-07-21T17:47:25Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:47:25Z" name: isvc-secondary-4e9225-predictor-7c68b4f8-5nwt5.18c45f37b0d1f3af namespace: kserve-ci-e2e-test resourceVersion: "20562" uid: 62da778c-24d1-4371-b149-1a802f27f297 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:47:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-4e9225-predictor-7c68b4f8-5nwt5 namespace: kserve-ci-e2e-test resourceVersion: "20548" uid: 07b0984c-5c5f-453d-8f6a-018d0e6e3d02 kind: Event lastTimestamp: "2026-07-21T17:47:29Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e12dde312cd424230e1ef8053d8890ee8a23543039767c0a577fe647c927b0a" already present on machine metadata: creationTimestamp: "2026-07-21T17:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:29Z" name: isvc-secondary-4e9225-predictor-7c68b4f8-5nwt5.18c45f37b2929b64 namespace: kserve-ci-e2e-test resourceVersion: "20603" uid: e4f3b3b5-090b-4bc7-b989-a6a25c1718c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:47:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-4e9225-predictor-7c68b4f8-5nwt5 namespace: kserve-ci-e2e-test resourceVersion: "20548" uid: 07b0984c-5c5f-453d-8f6a-018d0e6e3d02 kind: Event lastTimestamp: "2026-07-21T17:47:29Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-21T17:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:29Z" name: isvc-secondary-4e9225-predictor-7c68b4f8-5nwt5.18c45f37b5beca36 namespace: kserve-ci-e2e-test resourceVersion: "20606" uid: 9c57c04c-387c-4eeb-845c-cb8d82a5f848 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:47:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-4e9225-predictor-7c68b4f8-5nwt5 namespace: kserve-ci-e2e-test resourceVersion: "20548" uid: 07b0984c-5c5f-453d-8f6a-018d0e6e3d02 kind: Event lastTimestamp: "2026-07-21T17:47:29Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-21T17:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:29Z" name: isvc-secondary-4e9225-predictor-7c68b4f8-5nwt5.18c45f37b62e6f44 namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: e00bbdb1-d1a6-484f-8d0b-b709fd25450f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-4e9225-predictor-7c68b4f8-5nwt5 namespace: kserve-ci-e2e-test resourceVersion: "20548" uid: 07b0984c-5c5f-453d-8f6a-018d0e6e3d02 kind: Event lastTimestamp: "2026-07-21T17:47:34Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-4e9225-predictor-7c68b4f8-5nwt5_kserve-ci-e2e-test(07b0984c-5c5f-453d-8f6a-018d0e6e3d02) metadata: creationTimestamp: "2026-07-21T17:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:34Z" name: isvc-secondary-4e9225-predictor-7c68b4f8-5nwt5.18c45f39e1ea7281 namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: e16881dc-9625-46fb-affc-f53648bdfea9 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-4e9225-predictor-7c68b4f8 namespace: kserve-ci-e2e-test resourceVersion: "20535" uid: 88f2cf93-25bf-472e-823c-758dd92f062c kind: Event lastTimestamp: "2026-07-21T17:47:24Z" message: 'Created pod: isvc-secondary-4e9225-predictor-7c68b4f8-5nwt5' metadata: creationTimestamp: "2026-07-21T17:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:47:24Z" name: isvc-secondary-4e9225-predictor-7c68b4f8.18c45f3796ce8e6f namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: c8cb0bc1-b6bb-4eb8-8eec-c91eeabdc44a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-4e9225-predictor namespace: kserve-ci-e2e-test resourceVersion: "20533" uid: 697c2686-402c-4d15-930e-38518ccfb5c2 kind: Event lastTimestamp: "2026-07-21T17:47:24Z" message: Scaled up replica set isvc-secondary-4e9225-predictor-7c68b4f8 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:47:24Z" name: isvc-secondary-4e9225-predictor.18c45f37929afff7 namespace: kserve-ci-e2e-test resourceVersion: "20537" uid: bcc9d780-adfe-4944-949f-11ef7ee2effa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-4e9225-predictor namespace: kserve-ci-e2e-test resourceVersion: "20534" uid: 72e1adfd-bee9-454f-b468-a1587c0b0674 kind: Event lastTimestamp: "2026-07-21T17:47:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:47:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:47:39Z" name: isvc-secondary-4e9225-predictor.18c45f3b1151c658 namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 8369a6fa-095b-4884-94f5-e40ab9587533 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-4e9225-predictor namespace: kserve-ci-e2e-test resourceVersion: "20534" uid: 72e1adfd-bee9-454f-b468-a1587c0b0674 kind: Event lastTimestamp: "2026-07-21T17:47:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:47:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:47:39Z" name: isvc-secondary-4e9225-predictor.18c45f3b11be596c namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: 60e4396e-47b4-4759-81a9-660f79997bed reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-21T17:47:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-4e9225 namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: b063039f-6460-4b01-baca-33205b44dac8 kind: Event lastTimestamp: "2026-07-21T17:47:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:47:34Z" name: isvc-secondary-4e9225.18c45f3792987bc8 namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: 87a9fb8e-9ccc-4a9a-a959-e2d4d5db1073 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:47:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-4e9225 namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: b063039f-6460-4b01-baca-33205b44dac8 kind: Event lastTimestamp: "2026-07-21T17:47:24Z" message: 'Failed to update status for InferenceService "isvc-secondary-4e9225": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-4e9225": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:47:24Z" name: isvc-secondary-4e9225.18c45f3792ed8885 namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: dfe502bc-6124-4757-8fed-c215a0ff2745 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:47:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-4e9225 namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: b063039f-6460-4b01-baca-33205b44dac8 kind: Event lastTimestamp: "2026-07-21T17:47:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-4e9225": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:47:24Z" name: isvc-secondary-4e9225.18c45f3792edc8d3 namespace: kserve-ci-e2e-test resourceVersion: "20557" uid: 58a76042-444a-4270-a9f7-34878cd545c7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:37:18.204618Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: 4ef01e8e-1dba-4480-a64a-5edabe558e54 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs to ip-10-0-136-31.ec2.internal metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:37:18Z" name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs.18c45eaa60a7e1d1 namespace: kserve-ci-e2e-test resourceVersion: "15382" uid: b3fdbf82-36ed-440e-8487-0c9a95246acb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf7f8bd85-gq8p9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:18Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs namespace: kserve-ci-e2e-test resourceVersion: "15391" uid: 4ef01e8e-1dba-4480-a64a-5edabe558e54 kind: Event lastTimestamp: "2026-07-21T17:37:18Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:37:18Z" name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs.18c45eaa7a161116 namespace: kserve-ci-e2e-test resourceVersion: "15425" uid: 2df00d94-b11a-4768-a3d4-ec7feebcab0a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: 4ef01e8e-1dba-4480-a64a-5edabe558e54 kind: Event lastTimestamp: "2026-07-21T17:37:18Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e12dde312cd424230e1ef8053d8890ee8a23543039767c0a577fe647c927b0a" already present on machine metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:18Z" name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs.18c45eaa7b9c0797 namespace: kserve-ci-e2e-test resourceVersion: "15427" uid: 7a1e40c3-03d5-4586-875d-064d871244aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: 4ef01e8e-1dba-4480-a64a-5edabe558e54 kind: Event lastTimestamp: "2026-07-21T17:37:18Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:18Z" name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs.18c45eaa7f7192ae namespace: kserve-ci-e2e-test resourceVersion: "15428" uid: c98c4cfa-9fc2-4704-bbe0-f471db38d7f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: 4ef01e8e-1dba-4480-a64a-5edabe558e54 kind: Event lastTimestamp: "2026-07-21T17:37:18Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:18Z" name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs.18c45eaa7ffa6a56 namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 17d22da8-5c4b-4632-bcee-d7b2e3c02b8a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: 4ef01e8e-1dba-4480-a64a-5edabe558e54 kind: Event lastTimestamp: "2026-07-21T17:37:22Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:22Z" name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs.18c45eab74f27784 namespace: kserve-ci-e2e-test resourceVersion: "15477" uid: 49569a88-948f-461d-ab9a-b7ed076723fc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: 4ef01e8e-1dba-4480-a64a-5edabe558e54 kind: Event lastTimestamp: "2026-07-21T17:37:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:22Z" name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs.18c45eab78d2f3a3 namespace: kserve-ci-e2e-test resourceVersion: "15482" uid: c109d8fd-366e-4c4a-a0fb-e2be54189e30 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: 4ef01e8e-1dba-4480-a64a-5edabe558e54 kind: Event lastTimestamp: "2026-07-21T17:37:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:22Z" name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs.18c45eab794afc70 namespace: kserve-ci-e2e-test resourceVersion: "15483" uid: ce1b34dc-f337-44f6-8639-f0478dbb3102 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: 4ef01e8e-1dba-4480-a64a-5edabe558e54 kind: Event lastTimestamp: "2026-07-21T17:37:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:22Z" name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs.18c45eab79569a2c namespace: kserve-ci-e2e-test resourceVersion: "15484" uid: 47ad7b27-80a7-4e73-93c2-25f378981577 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: 4ef01e8e-1dba-4480-a64a-5edabe558e54 kind: Event lastTimestamp: "2026-07-21T17:37:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:22Z" name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs.18c45eab7c6b3c2b namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 0546f891-5507-4571-8603-9070d6c54bb5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: 4ef01e8e-1dba-4480-a64a-5edabe558e54 kind: Event lastTimestamp: "2026-07-21T17:37:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:22Z" name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs.18c45eab7cd924da namespace: kserve-ci-e2e-test resourceVersion: "15490" uid: 71ed12ac-ffd6-4715-8334-b5d8ed1de727 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-21T17:37:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: 4ef01e8e-1dba-4480-a64a-5edabe558e54 kind: Event lastTimestamp: "2026-07-21T17:39:10Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:10Z" name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs.18c45eabed398681 namespace: kserve-ci-e2e-test resourceVersion: "16501" uid: a0368fb9-d42e-4a68-9692-d97eadb21ddc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: 4ef01e8e-1dba-4480-a64a-5edabe558e54 kind: Event lastTimestamp: "2026-07-21T17:39:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:08Z" name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs.18c45ec40a99f77e namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 1a4365d2-1ee9-47a6-a1df-e771810c5096 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: 4ef01e8e-1dba-4480-a64a-5edabe558e54 kind: Event lastTimestamp: "2026-07-21T17:39:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:08Z" name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs.18c45ec40a9aadf6 namespace: kserve-ci-e2e-test resourceVersion: "16382" uid: 673edf9a-8394-494e-91e0-dc4fc4a39ab5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: 4ef01e8e-1dba-4480-a64a-5edabe558e54 kind: Event lastTimestamp: "2026-07-21T17:39:10Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:39:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:10Z" name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs.18c45ec49b95b357 namespace: kserve-ci-e2e-test resourceVersion: "16499" uid: 6afb761a-cf97-4da6-88bb-aa3779ddc222 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8 namespace: kserve-ci-e2e-test resourceVersion: "15358" uid: 318db81b-dbe3-4b5a-a2cd-2162054e20b4 kind: Event lastTimestamp: "2026-07-21T17:37:18Z" message: 'Created pod: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8-c46cs' metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:18Z" name: isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8.18c45eaa5f1845eb namespace: kserve-ci-e2e-test resourceVersion: "15375" uid: 17b414b7-6644-46cf-9fc8-a84ddf5ab645 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-69373-predictor namespace: kserve-ci-e2e-test resourceVersion: "15353" uid: bc6c88fd-43c4-4437-8d64-b50abbf3551f kind: Event lastTimestamp: "2026-07-21T17:37:18Z" message: Scaled up replica set isvc-sklearn-graph-raw-69373-predictor-567f6dfdd8 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:18Z" name: isvc-sklearn-graph-raw-69373-predictor.18c45eaa596e7971 namespace: kserve-ci-e2e-test resourceVersion: "15360" uid: c38adb6c-4d12-46c1-82a4-02e62ec06b16 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:37:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-69373-predictor namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 38a21b91-fe79-4c84-96c0-cac54190b983 kind: Event lastTimestamp: "2026-07-21T17:37:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:37:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:48Z" name: isvc-sklearn-graph-raw-69373-predictor.18c45eadd88ba5c4 namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: f5e83e31-8445-45b0-85bd-df082742d8e4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:37:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-69373-predictor namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 38a21b91-fe79-4c84-96c0-cac54190b983 kind: Event lastTimestamp: "2026-07-21T17:37:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:37:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:48Z" name: isvc-sklearn-graph-raw-69373-predictor.18c45eadd92026a8 namespace: kserve-ci-e2e-test resourceVersion: "15691" uid: 7da439bd-404d-4acb-9f08-ffdd6c620b25 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-21T17:38:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-69373-predictor namespace: kserve-ci-e2e-test resourceVersion: "15571" uid: 38a21b91-fe79-4c84-96c0-cac54190b983 kind: Event lastTimestamp: "2026-07-21T17:39:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:03Z" name: isvc-sklearn-graph-raw-69373-predictor.18c45eb4d77338f7 namespace: kserve-ci-e2e-test resourceVersion: "16288" uid: 019c974a-533f-480a-94cc-c2f8ce49749a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-21T17:38:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-69373-predictor namespace: kserve-ci-e2e-test resourceVersion: "15571" uid: 38a21b91-fe79-4c84-96c0-cac54190b983 kind: Event lastTimestamp: "2026-07-21T17:39:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:03Z" name: isvc-sklearn-graph-raw-69373-predictor.18c45eb4d7daa58e namespace: kserve-ci-e2e-test resourceVersion: "16289" uid: 8f14e35c-bf25-4f5e-a94a-9ca5950770d1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-21T17:37:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-69373 namespace: kserve-ci-e2e-test resourceVersion: "15332" uid: e3b994af-6444-4609-a014-b2027ccd5583 kind: Event lastTimestamp: "2026-07-21T17:38:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:38:40Z" name: isvc-sklearn-graph-raw-69373.18c45eaa5971d708 namespace: kserve-ci-e2e-test resourceVersion: "16097" uid: f0408f7d-00a4-40e6-8542-137980ac6ae1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-69373 namespace: kserve-ci-e2e-test resourceVersion: "15332" uid: e3b994af-6444-4609-a014-b2027ccd5583 kind: Event lastTimestamp: "2026-07-21T17:37:18Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-69373": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-69373": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:37:18Z" name: isvc-sklearn-graph-raw-69373.18c45eaa5a197774 namespace: kserve-ci-e2e-test resourceVersion: "15366" uid: 1257d7d6-5262-409c-a705-2d3fb0b06bfa reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-69373 namespace: kserve-ci-e2e-test resourceVersion: "16095" uid: e3b994af-6444-4609-a014-b2027ccd5583 kind: Event lastTimestamp: "2026-07-21T17:38:40Z" message: InferenceService [isvc-sklearn-graph-raw-69373] is Ready metadata: creationTimestamp: "2026-07-21T17:38:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:38:40Z" name: isvc-sklearn-graph-raw-69373.18c45ebda50d5112 namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: ff3eb58d-b9d0-40ce-b2e4-88f8b218978d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:39:08.487167Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c namespace: kserve-ci-e2e-test resourceVersion: "16389" uid: 54dbcf02-71e1-4da0-9b91-184e0f11697b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c to ip-10-0-136-31.ec2.internal metadata: creationTimestamp: "2026-07-21T17:39:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:39:08Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c.18c45ec40e021901 namespace: kserve-ci-e2e-test resourceVersion: "16399" uid: 9d65fc5c-bbaa-40ff-8a89-7c59180bf217 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf7f8bd85-gq8p9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:08Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 54dbcf02-71e1-4da0-9b91-184e0f11697b kind: Event lastTimestamp: "2026-07-21T17:39:08Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-hpa-3ad06-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:08Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c.18c45ec41cd2e5c7 namespace: kserve-ci-e2e-test resourceVersion: "16454" uid: 6700b3eb-3b6e-4a23-bf48-971b53815160 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:09Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c namespace: kserve-ci-e2e-test resourceVersion: "16412" uid: 54dbcf02-71e1-4da0-9b91-184e0f11697b kind: Event lastTimestamp: "2026-07-21T17:39:09Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:39:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:39:09Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c.18c45ec44b22932a namespace: kserve-ci-e2e-test resourceVersion: "16471" uid: b40a192a-7163-4d79-8583-a124d5ded16e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 54dbcf02-71e1-4da0-9b91-184e0f11697b kind: Event lastTimestamp: "2026-07-21T17:39:09Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e12dde312cd424230e1ef8053d8890ee8a23543039767c0a577fe647c927b0a" already present on machine metadata: creationTimestamp: "2026-07-21T17:39:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:09Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c.18c45ec44c9a4fef namespace: kserve-ci-e2e-test resourceVersion: "16474" uid: 3cddbff4-a7fb-4c96-9670-50fea4033d13 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 54dbcf02-71e1-4da0-9b91-184e0f11697b kind: Event lastTimestamp: "2026-07-21T17:39:09Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-21T17:39:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:09Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c.18c45ec44f89aa30 namespace: kserve-ci-e2e-test resourceVersion: "16476" uid: cab06e63-2969-4343-9e4a-8b33ed971996 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 54dbcf02-71e1-4da0-9b91-184e0f11697b kind: Event lastTimestamp: "2026-07-21T17:39:09Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-21T17:39:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:09Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c.18c45ec4503a4bd1 namespace: kserve-ci-e2e-test resourceVersion: "16477" uid: b6f58fe9-0513-4ed8-8cf0-c387e5dded6a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 54dbcf02-71e1-4da0-9b91-184e0f11697b kind: Event lastTimestamp: "2026-07-21T17:39:14Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:14Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c.18c45ec56d2b07d1 namespace: kserve-ci-e2e-test resourceVersion: "16539" uid: 0866584e-5337-4e9a-afb4-12c063288b5c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 54dbcf02-71e1-4da0-9b91-184e0f11697b kind: Event lastTimestamp: "2026-07-21T17:39:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:14Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c.18c45ec57022ab09 namespace: kserve-ci-e2e-test resourceVersion: "16541" uid: d515e76e-c5df-4ff6-87bd-a0d159125dcf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 54dbcf02-71e1-4da0-9b91-184e0f11697b kind: Event lastTimestamp: "2026-07-21T17:39:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:14Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c.18c45ec5709b877e namespace: kserve-ci-e2e-test resourceVersion: "16542" uid: 47058d48-a74c-476b-b348-ffe738e8265b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 54dbcf02-71e1-4da0-9b91-184e0f11697b kind: Event lastTimestamp: "2026-07-21T17:39:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:14Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c.18c45ec570a59755 namespace: kserve-ci-e2e-test resourceVersion: "16544" uid: ec4205c8-7c24-45a6-990a-c2effc00cfe2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 54dbcf02-71e1-4da0-9b91-184e0f11697b kind: Event lastTimestamp: "2026-07-21T17:39:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:14Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c.18c45ec5739a4ade namespace: kserve-ci-e2e-test resourceVersion: "16547" uid: 43241b9b-57d7-409e-a831-613a9b5bd800 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 54dbcf02-71e1-4da0-9b91-184e0f11697b kind: Event lastTimestamp: "2026-07-21T17:39:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:14Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c.18c45ec5740a4f47 namespace: kserve-ci-e2e-test resourceVersion: "16549" uid: d09be60c-1494-4943-ac18-8b14b49ff747 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-21T17:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 54dbcf02-71e1-4da0-9b91-184e0f11697b kind: Event lastTimestamp: "2026-07-21T17:41:01Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:01Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c.18c45ec5a9326c4d namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 797c737b-1e74-4396-bcd1-dc7e0e6c3599 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 54dbcf02-71e1-4da0-9b91-184e0f11697b kind: Event lastTimestamp: "2026-07-21T17:40:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:40:58Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c.18c45eddbd0f3818 namespace: kserve-ci-e2e-test resourceVersion: "17435" uid: fecc1e8b-263b-4edd-821b-c4981b70c48b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 54dbcf02-71e1-4da0-9b91-184e0f11697b kind: Event lastTimestamp: "2026-07-21T17:40:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:40:58Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c.18c45eddbd0fabe0 namespace: kserve-ci-e2e-test resourceVersion: "17436" uid: 0a064c65-10c8-40f6-a353-cadc9a31314a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 54dbcf02-71e1-4da0-9b91-184e0f11697b kind: Event lastTimestamp: "2026-07-21T17:41:01Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:41:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:01Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c.18c45ede57840fc5 namespace: kserve-ci-e2e-test resourceVersion: "17505" uid: 472561d0-b2ea-46b3-9ec3-b9a228be1cac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76 namespace: kserve-ci-e2e-test resourceVersion: "16368" uid: 496f3cbe-8fea-4a34-8f13-974a87aaa8c9 kind: Event lastTimestamp: "2026-07-21T17:39:08Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76-nrc9c' metadata: creationTimestamp: "2026-07-21T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:08Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76.18c45ec40d6dc3a1 namespace: kserve-ci-e2e-test resourceVersion: "16392" uid: 47546846-5f57-441a-bd70-a75214571a0c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-3ad06-predictor namespace: kserve-ci-e2e-test resourceVersion: "16360" uid: 08fc6e1b-35e9-4dfc-acb7-447d392f96b2 kind: Event lastTimestamp: "2026-07-21T17:39:08Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-3ad06-predictor-87ff8fc76 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:08Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor.18c45ec4092cb0af namespace: kserve-ci-e2e-test resourceVersion: "16371" uid: e31e7476-c801-448f-b5f9-a4d274c14625 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:39:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3ad06-predictor namespace: kserve-ci-e2e-test resourceVersion: "16366" uid: 077f7e2c-de3d-4bbb-ae84-818a815c4fec kind: Event lastTimestamp: "2026-07-21T17:39:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:39:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:53Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor.18c45ec787ba49ce namespace: kserve-ci-e2e-test resourceVersion: "16850" uid: 4b1a1737-fd2d-40fd-ae6e-61f205c912f6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:39:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3ad06-predictor namespace: kserve-ci-e2e-test resourceVersion: "16366" uid: 077f7e2c-de3d-4bbb-ae84-818a815c4fec kind: Event lastTimestamp: "2026-07-21T17:39:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:39:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:53Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor.18c45ec788383b87 namespace: kserve-ci-e2e-test resourceVersion: "16851" uid: b38ecfc1-9fc6-4e67-9f58-6df64e175d26 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-21T17:40:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3ad06-predictor namespace: kserve-ci-e2e-test resourceVersion: "16614" uid: 077f7e2c-de3d-4bbb-ae84-818a815c4fec kind: Event lastTimestamp: "2026-07-21T17:40:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:40:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:40:53Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor.18c45ed207225501 namespace: kserve-ci-e2e-test resourceVersion: "17340" uid: 012cbe2e-0867-4807-b97d-637d15e08280 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-21T17:40:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3ad06-predictor namespace: kserve-ci-e2e-test resourceVersion: "16614" uid: 077f7e2c-de3d-4bbb-ae84-818a815c4fec kind: Event lastTimestamp: "2026-07-21T17:40:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:40:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:40:53Z" name: isvc-sklearn-graph-raw-hpa-3ad06-predictor.18c45ed2078b12df namespace: kserve-ci-e2e-test resourceVersion: "17341" uid: de2ab532-0684-4507-a86b-7307ba449dbe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-21T17:39:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-3ad06 namespace: kserve-ci-e2e-test resourceVersion: "16352" uid: 1b52f1ad-f8ca-462a-a76e-6f9849f51b98 kind: Event lastTimestamp: "2026-07-21T17:40:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:40:31Z" name: isvc-sklearn-graph-raw-hpa-3ad06.18c45ec4092b0fc4 namespace: kserve-ci-e2e-test resourceVersion: "17145" uid: 097a5bcf-ef36-4b28-85b9-a05da880c4e3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:40:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-3ad06 namespace: kserve-ci-e2e-test resourceVersion: "17143" uid: 1b52f1ad-f8ca-462a-a76e-6f9849f51b98 kind: Event lastTimestamp: "2026-07-21T17:40:31Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-3ad06] is Ready metadata: creationTimestamp: "2026-07-21T17:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:40:31Z" name: isvc-sklearn-graph-raw-hpa-3ad06.18c45ed760d4329f namespace: kserve-ci-e2e-test resourceVersion: "17144" uid: f70e71ba-45ea-4b84-868c-40911ad00590 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:42:54.176453Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb namespace: kserve-ci-e2e-test resourceVersion: "18449" uid: 7b15d8fb-fd37-4092-a82d-855f29707a03 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb to ip-10-0-136-31.ec2.internal metadata: creationTimestamp: "2026-07-21T17:42:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:42:54Z" name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb.18c45ef89a233f47 namespace: kserve-ci-e2e-test resourceVersion: "18453" uid: f9047965-555b-4a06-8aa9-6f35a8e0108d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf7f8bd85-gq8p9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:42:54Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb namespace: kserve-ci-e2e-test resourceVersion: "18451" uid: 7b15d8fb-fd37-4092-a82d-855f29707a03 kind: Event lastTimestamp: "2026-07-21T17:42:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-90cae-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:42:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:42:54Z" name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb.18c45ef8a28a99a8 namespace: kserve-ci-e2e-test resourceVersion: "18481" uid: 8e6b0168-abfd-48c4-8658-9e125b359a71 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:42:55Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb namespace: kserve-ci-e2e-test resourceVersion: "18468" uid: 7b15d8fb-fd37-4092-a82d-855f29707a03 kind: Event lastTimestamp: "2026-07-21T17:42:55Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:42:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:42:55Z" name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb.18c45ef8d79d87cf namespace: kserve-ci-e2e-test resourceVersion: "18494" uid: d5675d60-d7c0-4991-a3a2-92f783de4204 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:42:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb namespace: kserve-ci-e2e-test resourceVersion: "18451" uid: 7b15d8fb-fd37-4092-a82d-855f29707a03 kind: Event lastTimestamp: "2026-07-21T17:42:55Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e12dde312cd424230e1ef8053d8890ee8a23543039767c0a577fe647c927b0a" already present on machine metadata: creationTimestamp: "2026-07-21T17:42:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:42:55Z" name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb.18c45ef8d90a9ba0 namespace: kserve-ci-e2e-test resourceVersion: "18496" uid: b452e4cf-2378-4410-9986-3a24550e36f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:42:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb namespace: kserve-ci-e2e-test resourceVersion: "18451" uid: 7b15d8fb-fd37-4092-a82d-855f29707a03 kind: Event lastTimestamp: "2026-07-21T17:42:55Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-21T17:42:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:42:55Z" name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb.18c45ef8dc9c6820 namespace: kserve-ci-e2e-test resourceVersion: "18497" uid: d13187c3-9bda-41a5-8126-9b58605c5405 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:42:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb namespace: kserve-ci-e2e-test resourceVersion: "18451" uid: 7b15d8fb-fd37-4092-a82d-855f29707a03 kind: Event lastTimestamp: "2026-07-21T17:42:55Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-21T17:42:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:42:55Z" name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb.18c45ef8dd0c7c03 namespace: kserve-ci-e2e-test resourceVersion: "18498" uid: cd12f2b6-970a-47fe-8900-733961f8ba8e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:42:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb namespace: kserve-ci-e2e-test resourceVersion: "18451" uid: 7b15d8fb-fd37-4092-a82d-855f29707a03 kind: Event lastTimestamp: "2026-07-21T17:42:59Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:42:59Z" name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb.18c45ef9ceadd1f5 namespace: kserve-ci-e2e-test resourceVersion: "18539" uid: c1fe06ef-201a-4ac1-8b66-c9a752f4dc48 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:42:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb namespace: kserve-ci-e2e-test resourceVersion: "18451" uid: 7b15d8fb-fd37-4092-a82d-855f29707a03 kind: Event lastTimestamp: "2026-07-21T17:42:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:42:59Z" name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb.18c45ef9d2a8a2bb namespace: kserve-ci-e2e-test resourceVersion: "18541" uid: 90707384-2c16-415a-85f7-8f320c143f71 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:42:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb namespace: kserve-ci-e2e-test resourceVersion: "18451" uid: 7b15d8fb-fd37-4092-a82d-855f29707a03 kind: Event lastTimestamp: "2026-07-21T17:42:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:42:59Z" name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb.18c45ef9d315d9c4 namespace: kserve-ci-e2e-test resourceVersion: "18544" uid: de8f060e-3c55-4449-b457-96185c3ec8c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:42:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb namespace: kserve-ci-e2e-test resourceVersion: "18451" uid: 7b15d8fb-fd37-4092-a82d-855f29707a03 kind: Event lastTimestamp: "2026-07-21T17:42:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:42:59Z" name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb.18c45ef9d31f428b namespace: kserve-ci-e2e-test resourceVersion: "18545" uid: de066831-ed09-47f1-bd6b-0b1599ffa106 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:42:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb namespace: kserve-ci-e2e-test resourceVersion: "18451" uid: 7b15d8fb-fd37-4092-a82d-855f29707a03 kind: Event lastTimestamp: "2026-07-21T17:42:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:42:59Z" name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb.18c45ef9d6908118 namespace: kserve-ci-e2e-test resourceVersion: "18547" uid: 17b278d5-9485-4681-8875-1411e28cae0a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:42:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb namespace: kserve-ci-e2e-test resourceVersion: "18451" uid: 7b15d8fb-fd37-4092-a82d-855f29707a03 kind: Event lastTimestamp: "2026-07-21T17:42:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:42:59Z" name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb.18c45ef9d6fd40c2 namespace: kserve-ci-e2e-test resourceVersion: "18548" uid: 2b76fa87-1409-460c-993c-44b32900c231 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 18 eventTime: null firstTimestamp: "2026-07-21T17:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb namespace: kserve-ci-e2e-test resourceVersion: "18451" uid: 7b15d8fb-fd37-4092-a82d-855f29707a03 kind: Event lastTimestamp: "2026-07-21T17:45:39Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:45:39Z" name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb.18c45efa46a6d9df namespace: kserve-ci-e2e-test resourceVersion: "19707" uid: 6ae0312f-f0e0-4582-b333-023f36a44a64 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:45:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb namespace: kserve-ci-e2e-test resourceVersion: "18451" uid: 7b15d8fb-fd37-4092-a82d-855f29707a03 kind: Event lastTimestamp: "2026-07-21T17:45:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:45:34Z" name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb.18c45f1de50a8d53 namespace: kserve-ci-e2e-test resourceVersion: "19673" uid: 2ba1d0bb-d374-4aea-9ef2-bcd6781c7456 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:45:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb namespace: kserve-ci-e2e-test resourceVersion: "18451" uid: 7b15d8fb-fd37-4092-a82d-855f29707a03 kind: Event lastTimestamp: "2026-07-21T17:45:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:45:34Z" name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb.18c45f1de50affa2 namespace: kserve-ci-e2e-test resourceVersion: "19674" uid: bc148404-30c2-431b-9840-f3671292e659 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:45:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb namespace: kserve-ci-e2e-test resourceVersion: "18451" uid: 7b15d8fb-fd37-4092-a82d-855f29707a03 kind: Event lastTimestamp: "2026-07-21T17:45:42Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:45:42Z" name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb.18c45f1e99469029 namespace: kserve-ci-e2e-test resourceVersion: "19780" uid: 88925d1a-83ae-4da1-bb96-94a09552f64d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:42:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4 namespace: kserve-ci-e2e-test resourceVersion: "18430" uid: 80b10647-d4a0-40d8-951f-1fa507be7df9 kind: Event lastTimestamp: "2026-07-21T17:42:54Z" message: 'Created pod: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4-mfqfb' metadata: creationTimestamp: "2026-07-21T17:42:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:42:54Z" name: isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4.18c45ef899b41b7f namespace: kserve-ci-e2e-test resourceVersion: "18450" uid: 9e055d35-6393-489d-bc98-63da988aa03a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:42:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-90cae-predictor namespace: kserve-ci-e2e-test resourceVersion: "18423" uid: e58e5353-015a-42e8-9a96-27774043f49d kind: Event lastTimestamp: "2026-07-21T17:42:54Z" message: Scaled up replica set isvc-sklearn-scale-raw-90cae-predictor-75cc6f54c4 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:42:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:42:54Z" name: isvc-sklearn-scale-raw-90cae-predictor.18c45ef895177bd8 namespace: kserve-ci-e2e-test resourceVersion: "18434" uid: 31a9bb7e-7d8b-4cf9-a4ab-fe2e87a89904 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:43:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-90cae-predictor namespace: kserve-ci-e2e-test resourceVersion: "18426" uid: 3d65ab68-05e0-4545-b259-498484194f06 kind: Event lastTimestamp: "2026-07-21T17:43:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:43:24Z" name: isvc-sklearn-scale-raw-90cae-predictor.18c45efc14301bb6 namespace: kserve-ci-e2e-test resourceVersion: "18740" uid: f4c8ebb2-4eb7-4f02-a2a8-233fe991c0c1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:43:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-90cae-predictor namespace: kserve-ci-e2e-test resourceVersion: "18426" uid: 3d65ab68-05e0-4545-b259-498484194f06 kind: Event lastTimestamp: "2026-07-21T17:43:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:43:24Z" name: isvc-sklearn-scale-raw-90cae-predictor.18c45efc149f3535 namespace: kserve-ci-e2e-test resourceVersion: "18741" uid: 1c10e40f-5d5d-4808-b23a-b6036fb66ec8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-21T17:43:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-90cae-predictor namespace: kserve-ci-e2e-test resourceVersion: "18627" uid: 3d65ab68-05e0-4545-b259-498484194f06 kind: Event lastTimestamp: "2026-07-21T17:45:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:45:24Z" name: isvc-sklearn-scale-raw-90cae-predictor.18c45f0312afe506 namespace: kserve-ci-e2e-test resourceVersion: "19574" uid: e9bcd139-221c-4b42-b798-98edac89643a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-21T17:43:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-90cae-predictor namespace: kserve-ci-e2e-test resourceVersion: "18627" uid: 3d65ab68-05e0-4545-b259-498484194f06 kind: Event lastTimestamp: "2026-07-21T17:45:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:45:24Z" name: isvc-sklearn-scale-raw-90cae-predictor.18c45f031319b687 namespace: kserve-ci-e2e-test resourceVersion: "19575" uid: ec526206-7a57-4a96-b2b7-b3ed493e7920 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-21T17:42:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-90cae namespace: kserve-ci-e2e-test resourceVersion: "18410" uid: 63ae660b-160c-4ebf-bfdc-53d7ebab5dfc kind: Event lastTimestamp: "2026-07-21T17:45:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:42:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:45:29Z" name: isvc-sklearn-scale-raw-90cae.18c45ef89511e0c0 namespace: kserve-ci-e2e-test resourceVersion: "19615" uid: 64787182-c8c9-4a03-8084-18722fb1e02b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:42:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-90cae namespace: kserve-ci-e2e-test resourceVersion: "18410" uid: 63ae660b-160c-4ebf-bfdc-53d7ebab5dfc kind: Event lastTimestamp: "2026-07-21T17:42:54Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-90cae": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-90cae": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:42:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:42:54Z" name: isvc-sklearn-scale-raw-90cae.18c45ef895b8b2be namespace: kserve-ci-e2e-test resourceVersion: "18473" uid: bcc6d788-954b-409d-8dc8-6958627ebeb9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:42:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-90cae namespace: kserve-ci-e2e-test resourceVersion: "18429" uid: 63ae660b-160c-4ebf-bfdc-53d7ebab5dfc kind: Event lastTimestamp: "2026-07-21T17:42:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-90cae": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:42:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:42:54Z" name: isvc-sklearn-scale-raw-90cae.18c45ef89c321ee5 namespace: kserve-ci-e2e-test resourceVersion: "18476" uid: d622e240-48bf-46c0-8eb0-55cb30d05849 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:45:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-90cae namespace: kserve-ci-e2e-test resourceVersion: "19613" uid: 63ae660b-160c-4ebf-bfdc-53d7ebab5dfc kind: Event lastTimestamp: "2026-07-21T17:45:29Z" message: InferenceService [isvc-sklearn-scale-raw-90cae] is Ready metadata: creationTimestamp: "2026-07-21T17:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:45:29Z" name: isvc-sklearn-scale-raw-90cae.18c45f1cac467e37 namespace: kserve-ci-e2e-test resourceVersion: "19614" uid: 2790a34f-f9aa-47e0-9187-741374dbf9d2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:37:18.325435Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: 06fd3a0c-6dbd-4e4c-b616-98b3ac8c2603 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w to ip-10-0-136-31.ec2.internal metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:37:18Z" name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w.18c45eaa67db6172 namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 438a4f96-52f2-4744-9240-617e80c95048 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf7f8bd85-gq8p9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:18Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 06fd3a0c-6dbd-4e4c-b616-98b3ac8c2603 kind: Event lastTimestamp: "2026-07-21T17:37:18Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:37:18Z" name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w.18c45eaa8177f595 namespace: kserve-ci-e2e-test resourceVersion: "15430" uid: 12b0eb4f-9971-403d-aaee-67df208a4671 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 06fd3a0c-6dbd-4e4c-b616-98b3ac8c2603 kind: Event lastTimestamp: "2026-07-21T17:37:18Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e12dde312cd424230e1ef8053d8890ee8a23543039767c0a577fe647c927b0a" already present on machine metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:18Z" name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w.18c45eaa82f0918b namespace: kserve-ci-e2e-test resourceVersion: "15433" uid: f76f1d09-4127-483c-95b0-35e27eba4fd9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 06fd3a0c-6dbd-4e4c-b616-98b3ac8c2603 kind: Event lastTimestamp: "2026-07-21T17:37:18Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:18Z" name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w.18c45eaa863dfcaf namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: 48339c1e-d7eb-4afb-b2b9-0f5a690ae790 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 06fd3a0c-6dbd-4e4c-b616-98b3ac8c2603 kind: Event lastTimestamp: "2026-07-21T17:37:18Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:18Z" name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w.18c45eaa86b2e53b namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: f38ee766-1dea-4061-a89f-c5931a1812ba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 06fd3a0c-6dbd-4e4c-b616-98b3ac8c2603 kind: Event lastTimestamp: "2026-07-21T17:37:22Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-21T17:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:22Z" name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w.18c45eab750dc243 namespace: kserve-ci-e2e-test resourceVersion: "15478" uid: 3e0978ff-cb3e-43d3-be75-1c40e6629ab8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 06fd3a0c-6dbd-4e4c-b616-98b3ac8c2603 kind: Event lastTimestamp: "2026-07-21T17:37:42Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.203s (19.203s including waiting). Image size: 1307347707 bytes.' metadata: creationTimestamp: "2026-07-21T17:37:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:42Z" name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w.18c45eafeda5355e namespace: kserve-ci-e2e-test resourceVersion: "15637" uid: 532d2386-999d-41d2-99e4-ad2bc9555be7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 06fd3a0c-6dbd-4e4c-b616-98b3ac8c2603 kind: Event lastTimestamp: "2026-07-21T17:37:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:37:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:42Z" name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w.18c45eaff0c639b3 namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: a06131d4-3cc1-4148-98df-417a57617f2d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 06fd3a0c-6dbd-4e4c-b616-98b3ac8c2603 kind: Event lastTimestamp: "2026-07-21T17:37:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:37:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:42Z" name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w.18c45eaff1382100 namespace: kserve-ci-e2e-test resourceVersion: "15639" uid: 77c38659-ff1d-44ce-b5ed-25a1d9580cfa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 06fd3a0c-6dbd-4e4c-b616-98b3ac8c2603 kind: Event lastTimestamp: "2026-07-21T17:37:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:37:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:42Z" name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w.18c45eaff1417931 namespace: kserve-ci-e2e-test resourceVersion: "15640" uid: 2b0cd364-7a4c-4eef-850f-f23abc8dbdc4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 06fd3a0c-6dbd-4e4c-b616-98b3ac8c2603 kind: Event lastTimestamp: "2026-07-21T17:37:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:37:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:42Z" name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w.18c45eaff4290c47 namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: a96dbda1-3cc4-4c2d-8030-e7607f37d051 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 06fd3a0c-6dbd-4e4c-b616-98b3ac8c2603 kind: Event lastTimestamp: "2026-07-21T17:37:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:37:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:42Z" name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w.18c45eaff49d2a54 namespace: kserve-ci-e2e-test resourceVersion: "15643" uid: e8ba16da-9d0e-4a98-9bc7-ee05aadc78dc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-21T17:37:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 06fd3a0c-6dbd-4e4c-b616-98b3ac8c2603 kind: Event lastTimestamp: "2026-07-21T17:39:08Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:37:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:08Z" name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w.18c45eb02382e39c namespace: kserve-ci-e2e-test resourceVersion: "16463" uid: 5d394b30-a69e-4f01-8e3b-c338e9fa2fe1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 06fd3a0c-6dbd-4e4c-b616-98b3ac8c2603 kind: Event lastTimestamp: "2026-07-21T17:39:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:08Z" name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w.18c45ec41914fc92 namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: ed1c87ba-c40b-4a49-81c5-5f98047f1186 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 06fd3a0c-6dbd-4e4c-b616-98b3ac8c2603 kind: Event lastTimestamp: "2026-07-21T17:39:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:08Z" name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w.18c45ec4191587d3 namespace: kserve-ci-e2e-test resourceVersion: "16450" uid: 9a84055a-452f-47bc-829e-8aec3bf3b3d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 06fd3a0c-6dbd-4e4c-b616-98b3ac8c2603 kind: Event lastTimestamp: "2026-07-21T17:39:08Z" message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp 10.133.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:08Z" name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w.18c45ec429aa5703 namespace: kserve-ci-e2e-test resourceVersion: "16462" uid: 0ffc66c9-c68c-4b27-a8cb-106563ba26b4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9 namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 08664f9f-ba60-4b06-a0c5-3d15dc223862 kind: Event lastTimestamp: "2026-07-21T17:37:18Z" message: 'Created pod: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9-8f67w' metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:18Z" name: isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9.18c45eaa6748d21c namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: 3e150468-aa56-4684-9c97-9c3a32a1f9ac reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-69373-predictor namespace: kserve-ci-e2e-test resourceVersion: "15388" uid: af7b9837-496c-4628-8089-fb4422f2cc9e kind: Event lastTimestamp: "2026-07-21T17:37:18Z" message: Scaled up replica set isvc-xgboost-graph-raw-69373-predictor-5dd484c6b9 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:18Z" name: isvc-xgboost-graph-raw-69373-predictor.18c45eaa64486dc7 namespace: kserve-ci-e2e-test resourceVersion: "15395" uid: 3f15fc88-2f0d-4637-9b03-10374813f2cb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:37:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-69373-predictor namespace: kserve-ci-e2e-test resourceVersion: "15390" uid: 6e99574d-24ca-422b-a6a4-f61e3cebb629 kind: Event lastTimestamp: "2026-07-21T17:37:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:37:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:48Z" name: isvc-xgboost-graph-raw-69373-predictor.18c45eade2bc7b4b namespace: kserve-ci-e2e-test resourceVersion: "15694" uid: 47a31eb9-2fb7-4548-b9ce-61eea57535d8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:37:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-69373-predictor namespace: kserve-ci-e2e-test resourceVersion: "15390" uid: 6e99574d-24ca-422b-a6a4-f61e3cebb629 kind: Event lastTimestamp: "2026-07-21T17:37:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:37:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:48Z" name: isvc-xgboost-graph-raw-69373-predictor.18c45eade329e6a0 namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: 17ecc02a-0587-49b3-9fbe-53ca4b845b0f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-21T17:38:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-69373-predictor namespace: kserve-ci-e2e-test resourceVersion: "15577" uid: 6e99574d-24ca-422b-a6a4-f61e3cebb629 kind: Event lastTimestamp: "2026-07-21T17:39:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:03Z" name: isvc-xgboost-graph-raw-69373-predictor.18c45eb4e0e6f75a namespace: kserve-ci-e2e-test resourceVersion: "16292" uid: 951b91e7-9d40-4565-8444-7fb2eb177064 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-21T17:38:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-69373-predictor namespace: kserve-ci-e2e-test resourceVersion: "15577" uid: 6e99574d-24ca-422b-a6a4-f61e3cebb629 kind: Event lastTimestamp: "2026-07-21T17:39:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:03Z" name: isvc-xgboost-graph-raw-69373-predictor.18c45eb4e158b28c namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: e07c1db3-989b-4ed5-bc64-642f143cd4b2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-21T17:37:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-69373 namespace: kserve-ci-e2e-test resourceVersion: "15372" uid: d6ea9edb-bf54-4c44-b4da-6d8d6f130be8 kind: Event lastTimestamp: "2026-07-21T17:38:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:38:39Z" name: isvc-xgboost-graph-raw-69373.18c45eaa6419b5b2 namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: 0e4b651e-cff7-43d4-a1ef-ccb67e1fae62 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-69373 namespace: kserve-ci-e2e-test resourceVersion: "15372" uid: d6ea9edb-bf54-4c44-b4da-6d8d6f130be8 kind: Event lastTimestamp: "2026-07-21T17:37:18Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-69373": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-69373": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:37:18Z" name: isvc-xgboost-graph-raw-69373.18c45eaa64832c03 namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: bbca11ac-5f8b-4864-8d73-5e98e0d215f7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-69373 namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: d6ea9edb-bf54-4c44-b4da-6d8d6f130be8 kind: Event lastTimestamp: "2026-07-21T17:38:39Z" message: InferenceService [isvc-xgboost-graph-raw-69373] is Ready metadata: creationTimestamp: "2026-07-21T17:38:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:38:39Z" name: isvc-xgboost-graph-raw-69373.18c45ebd333a9c00 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: a54cb9b7-c292-4f13-b910-b1a972b93a47 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:39:08.564797Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98 namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: d061a8db-98da-4a79-a3e6-e9a24df6ec04 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98 to ip-10-0-139-6.ec2.internal metadata: creationTimestamp: "2026-07-21T17:39:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:39:08Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98.18c45ec412a2980d namespace: kserve-ci-e2e-test resourceVersion: "16419" uid: b348df3d-f6d1-425b-b95b-bf2ea3ad55d0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf7f8bd85-gq8p9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:08Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98 namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: d061a8db-98da-4a79-a3e6-e9a24df6ec04 kind: Event lastTimestamp: "2026-07-21T17:39:08Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:39:08Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98.18c45ec42c59efea namespace: kserve-ci-e2e-test resourceVersion: "16464" uid: 220bed8b-5635-4734-b140-c73d08b3d561 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98 namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: d061a8db-98da-4a79-a3e6-e9a24df6ec04 kind: Event lastTimestamp: "2026-07-21T17:39:09Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e12dde312cd424230e1ef8053d8890ee8a23543039767c0a577fe647c927b0a" already present on machine metadata: creationTimestamp: "2026-07-21T17:39:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:09Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98.18c45ec42df29dc1 namespace: kserve-ci-e2e-test resourceVersion: "16466" uid: 56a211bd-482e-449d-a38e-b9a2edf3ca38 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-6.ec2.internal source: component: kubelet host: ip-10-0-139-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98 namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: d061a8db-98da-4a79-a3e6-e9a24df6ec04 kind: Event lastTimestamp: "2026-07-21T17:39:09Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-21T17:39:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:09Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98.18c45ec430e4b50c namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: e103fdb6-541f-4d5e-86ea-d64c1ed1904d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-6.ec2.internal source: component: kubelet host: ip-10-0-139-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98 namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: d061a8db-98da-4a79-a3e6-e9a24df6ec04 kind: Event lastTimestamp: "2026-07-21T17:39:09Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-21T17:39:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:09Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98.18c45ec4315dd977 namespace: kserve-ci-e2e-test resourceVersion: "16468" uid: 284e7dcb-e2cc-4736-a747-cebf8be6220c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-6.ec2.internal source: component: kubelet host: ip-10-0-139-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98 namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: d061a8db-98da-4a79-a3e6-e9a24df6ec04 kind: Event lastTimestamp: "2026-07-21T17:39:13Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-21T17:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:13Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98.18c45ec53fc0a03f namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: e2f48d77-81bf-4d27-8a8f-8ff315dc35df reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-6.ec2.internal source: component: kubelet host: ip-10-0-139-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98 namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: d061a8db-98da-4a79-a3e6-e9a24df6ec04 kind: Event lastTimestamp: "2026-07-21T17:39:32Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 18.955s (18.955s including waiting). Image size: 1307347707 bytes.' metadata: creationTimestamp: "2026-07-21T17:39:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:32Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98.18c45ec9a9997d5b namespace: kserve-ci-e2e-test resourceVersion: "16686" uid: dfa75d45-d08e-4641-b416-a8aff5f172f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-6.ec2.internal source: component: kubelet host: ip-10-0-139-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98 namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: d061a8db-98da-4a79-a3e6-e9a24df6ec04 kind: Event lastTimestamp: "2026-07-21T17:39:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:39:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:32Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98.18c45ec9accd7c5f namespace: kserve-ci-e2e-test resourceVersion: "16687" uid: bc803d7e-790d-481e-aae8-13551eef757b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-6.ec2.internal source: component: kubelet host: ip-10-0-139-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98 namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: d061a8db-98da-4a79-a3e6-e9a24df6ec04 kind: Event lastTimestamp: "2026-07-21T17:39:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:39:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:32Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98.18c45ec9ad480bfb namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: caa01b3f-22d3-4c60-beb7-180e9fc43513 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-6.ec2.internal source: component: kubelet host: ip-10-0-139-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98 namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: d061a8db-98da-4a79-a3e6-e9a24df6ec04 kind: Event lastTimestamp: "2026-07-21T17:39:32Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-21T17:39:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:32Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98.18c45ec9ad543cce namespace: kserve-ci-e2e-test resourceVersion: "16689" uid: e1812599-9b1d-4ab3-9c0f-d27072a124f1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-6.ec2.internal source: component: kubelet host: ip-10-0-139-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98 namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: d061a8db-98da-4a79-a3e6-e9a24df6ec04 kind: Event lastTimestamp: "2026-07-21T17:39:34Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.188s (2.188s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-21T17:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:34Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98.18c45eca2fc2c92c namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 3a7c759e-b061-4711-81ad-d50eca2c43e5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-6.ec2.internal source: component: kubelet host: ip-10-0-139-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98 namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: d061a8db-98da-4a79-a3e6-e9a24df6ec04 kind: Event lastTimestamp: "2026-07-21T17:39:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:34Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98.18c45eca3370aeaa namespace: kserve-ci-e2e-test resourceVersion: "16708" uid: 323f4bdd-e31a-487a-bd79-1513493836c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-6.ec2.internal source: component: kubelet host: ip-10-0-139-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98 namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: d061a8db-98da-4a79-a3e6-e9a24df6ec04 kind: Event lastTimestamp: "2026-07-21T17:39:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:34Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98.18c45eca33eafb1d namespace: kserve-ci-e2e-test resourceVersion: "16709" uid: 318aae7b-b970-4a5c-a96f-60974302fae7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-6.ec2.internal source: component: kubelet host: ip-10-0-139-6.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-21T17:39:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98 namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: d061a8db-98da-4a79-a3e6-e9a24df6ec04 kind: Event lastTimestamp: "2026-07-21T17:40:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:39:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:40:22Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98.18c45ecaa1871449 namespace: kserve-ci-e2e-test resourceVersion: "17073" uid: f9e6f890-3e77-4287-aaa4-4d02da25e5c4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-6.ec2.internal source: component: kubelet host: ip-10-0-139-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98 namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: d061a8db-98da-4a79-a3e6-e9a24df6ec04 kind: Event lastTimestamp: "2026-07-21T17:40:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:40:58Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98.18c45eddc5b18cb2 namespace: kserve-ci-e2e-test resourceVersion: "17471" uid: ecc7761a-9b75-4ea3-a264-be6fe82b456d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-6.ec2.internal source: component: kubelet host: ip-10-0-139-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98 namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: d061a8db-98da-4a79-a3e6-e9a24df6ec04 kind: Event lastTimestamp: "2026-07-21T17:40:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:40:58Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98.18c45eddc5b23c48 namespace: kserve-ci-e2e-test resourceVersion: "17473" uid: 6cde9257-3359-4cbb-b162-fa012a9314d9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-6.ec2.internal source: component: kubelet host: ip-10-0-139-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc namespace: kserve-ci-e2e-test resourceVersion: "16407" uid: cbbdfb59-bfbd-4705-9afe-3ab4fd0209fb kind: Event lastTimestamp: "2026-07-21T17:39:08Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc-zph98' metadata: creationTimestamp: "2026-07-21T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:08Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc.18c45ec41236e77a namespace: kserve-ci-e2e-test resourceVersion: "16416" uid: 7b394d22-f39d-4c6f-bd9c-8607e2049974 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-3ad06-predictor namespace: kserve-ci-e2e-test resourceVersion: "16404" uid: 4585b979-5195-4780-a661-02c31c9233ff kind: Event lastTimestamp: "2026-07-21T17:39:08Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-3ad06-predictor-dc9c5c7fc from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:08Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor.18c45ec40fa63c62 namespace: kserve-ci-e2e-test resourceVersion: "16409" uid: 861cde68-a004-4961-9fad-8a530fc01f5e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:39:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3ad06-predictor namespace: kserve-ci-e2e-test resourceVersion: "16406" uid: 45fc6f8e-06d9-445a-ac0b-3b1fbd64edbf kind: Event lastTimestamp: "2026-07-21T17:40:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:39:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:40:08Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor.18c45ec78e434ea1 namespace: kserve-ci-e2e-test resourceVersion: "16969" uid: 71b590a2-5321-48dc-94d3-43af62088633 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:39:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3ad06-predictor namespace: kserve-ci-e2e-test resourceVersion: "16406" uid: 45fc6f8e-06d9-445a-ac0b-3b1fbd64edbf kind: Event lastTimestamp: "2026-07-21T17:40:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:39:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:40:08Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor.18c45ec78ebd9c76 namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: c6f01366-2c5d-498d-8c3c-8bb326a3a3e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-21T17:39:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3ad06-predictor namespace: kserve-ci-e2e-test resourceVersion: "16617" uid: 45fc6f8e-06d9-445a-ac0b-3b1fbd64edbf kind: Event lastTimestamp: "2026-07-21T17:40:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:39:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:40:53Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor.18c45ece8cf27936 namespace: kserve-ci-e2e-test resourceVersion: "17342" uid: e2ccd8d4-b9df-40e1-9860-61004bee0469 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-21T17:39:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3ad06-predictor namespace: kserve-ci-e2e-test resourceVersion: "16617" uid: 45fc6f8e-06d9-445a-ac0b-3b1fbd64edbf kind: Event lastTimestamp: "2026-07-21T17:40:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:39:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:40:53Z" name: isvc-xgboost-graph-raw-hpa-3ad06-predictor.18c45ece8dadf7f7 namespace: kserve-ci-e2e-test resourceVersion: "17343" uid: 492c4bcf-ebd7-42d4-be3b-f0afcbeb850f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-21T17:39:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-3ad06 namespace: kserve-ci-e2e-test resourceVersion: "16384" uid: 1f377f95-4362-4264-90fb-1ff8d5d8ba6f kind: Event lastTimestamp: "2026-07-21T17:40:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:40:32Z" name: isvc-xgboost-graph-raw-hpa-3ad06.18c45ec40fa3411c namespace: kserve-ci-e2e-test resourceVersion: "17163" uid: ef085e16-fff1-47b8-99d8-b42af784576a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:40:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-3ad06 namespace: kserve-ci-e2e-test resourceVersion: "17161" uid: 1f377f95-4362-4264-90fb-1ff8d5d8ba6f kind: Event lastTimestamp: "2026-07-21T17:40:32Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-3ad06] is Ready metadata: creationTimestamp: "2026-07-21T17:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:40:32Z" name: isvc-xgboost-graph-raw-hpa-3ad06.18c45ed7b1724bb9 namespace: kserve-ci-e2e-test resourceVersion: "17162" uid: 13b8e325-4c99-455b-a789-a7bc3c16d1ab reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:40:58.822358Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg namespace: kserve-ci-e2e-test resourceVersion: "17437" uid: 1f080ef3-9e5b-4f7f-b17d-e8ff69f06b4c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg to ip-10-0-136-31.ec2.internal metadata: creationTimestamp: "2026-07-21T17:40:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:40:58Z" name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg.18c45eddbe7f795a namespace: kserve-ci-e2e-test resourceVersion: "17444" uid: a4a5607e-9c87-466c-b38e-f29fa8e314cb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf7f8bd85-gq8p9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:40:59Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg namespace: kserve-ci-e2e-test resourceVersion: "17452" uid: 1f080ef3-9e5b-4f7f-b17d-e8ff69f06b4c kind: Event lastTimestamp: "2026-07-21T17:40:59Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:40:59Z" name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg.18c45eddd85539e5 namespace: kserve-ci-e2e-test resourceVersion: "17480" uid: 9c281e60-2b8c-4f22-85b8-31f90bc64373 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:40:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg namespace: kserve-ci-e2e-test resourceVersion: "17439" uid: 1f080ef3-9e5b-4f7f-b17d-e8ff69f06b4c kind: Event lastTimestamp: "2026-07-21T17:40:59Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-21T17:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:40:59Z" name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg.18c45eddd9cdf912 namespace: kserve-ci-e2e-test resourceVersion: "17482" uid: 60e97fc9-cf75-4eaa-99ab-3695426ccba7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg namespace: kserve-ci-e2e-test resourceVersion: "17439" uid: 1f080ef3-9e5b-4f7f-b17d-e8ff69f06b4c kind: Event lastTimestamp: "2026-07-21T17:41:00Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.2s (1.2s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-21T17:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:00Z" name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg.18c45ede215e99a8 namespace: kserve-ci-e2e-test resourceVersion: "17489" uid: 8358f7df-98fb-4fe8-960e-2817f8c544e2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg namespace: kserve-ci-e2e-test resourceVersion: "17439" uid: 1f080ef3-9e5b-4f7f-b17d-e8ff69f06b4c kind: Event lastTimestamp: "2026-07-21T17:41:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:00Z" name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg.18c45ede24cf6b46 namespace: kserve-ci-e2e-test resourceVersion: "17490" uid: e1a72622-6e9b-4889-8db5-622c9e45291a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg namespace: kserve-ci-e2e-test resourceVersion: "17439" uid: 1f080ef3-9e5b-4f7f-b17d-e8ff69f06b4c kind: Event lastTimestamp: "2026-07-21T17:41:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:00Z" name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg.18c45ede253dc50f namespace: kserve-ci-e2e-test resourceVersion: "17492" uid: 91351225-51da-4b58-b50e-a0f608ef320f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg namespace: kserve-ci-e2e-test resourceVersion: "17439" uid: 1f080ef3-9e5b-4f7f-b17d-e8ff69f06b4c kind: Event lastTimestamp: "2026-07-21T17:41:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:00Z" name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg.18c45ede2547eef1 namespace: kserve-ci-e2e-test resourceVersion: "17493" uid: 193ab504-9112-40d4-bce7-5428055d4299 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg namespace: kserve-ci-e2e-test resourceVersion: "17439" uid: 1f080ef3-9e5b-4f7f-b17d-e8ff69f06b4c kind: Event lastTimestamp: "2026-07-21T17:41:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:00Z" name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg.18c45ede283110f7 namespace: kserve-ci-e2e-test resourceVersion: "17494" uid: 722733ad-c781-4fad-ae1c-5d365d293c36 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg namespace: kserve-ci-e2e-test resourceVersion: "17439" uid: 1f080ef3-9e5b-4f7f-b17d-e8ff69f06b4c kind: Event lastTimestamp: "2026-07-21T17:41:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:00Z" name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg.18c45ede289f52a5 namespace: kserve-ci-e2e-test resourceVersion: "17495" uid: d39fecd6-160c-4d44-9dbb-d4a4b0581b92 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:42:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg namespace: kserve-ci-e2e-test resourceVersion: "17439" uid: 1f080ef3-9e5b-4f7f-b17d-e8ff69f06b4c kind: Event lastTimestamp: "2026-07-21T17:42:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:42:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:42:54Z" name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg.18c45ef89ccef6d3 namespace: kserve-ci-e2e-test resourceVersion: "18474" uid: ec11f747-088f-48e1-bf07-1222aa468f13 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:42:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg namespace: kserve-ci-e2e-test resourceVersion: "17439" uid: 1f080ef3-9e5b-4f7f-b17d-e8ff69f06b4c kind: Event lastTimestamp: "2026-07-21T17:42:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:42:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:42:54Z" name: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg.18c45ef89ccf63a2 namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: 0e2dce4c-13dd-4260-b747-e7fa2c3d23ca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:40:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-4098a-predictor-f7bdb8d96 namespace: kserve-ci-e2e-test resourceVersion: "17416" uid: df537a48-e0af-462e-a6be-6b467c9803ad kind: Event lastTimestamp: "2026-07-21T17:40:58Z" message: 'Created pod: message-dumper-raw-4098a-predictor-f7bdb8d96-bkghg' metadata: creationTimestamp: "2026-07-21T17:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:40:58Z" name: message-dumper-raw-4098a-predictor-f7bdb8d96.18c45eddbdefe649 namespace: kserve-ci-e2e-test resourceVersion: "17438" uid: 8a745922-d634-4be1-a641-b4a209637bec reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:40:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-4098a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17412" uid: 4516697e-ef36-4dd5-a15e-f09a787a8ba0 kind: Event lastTimestamp: "2026-07-21T17:40:58Z" message: Scaled up replica set message-dumper-raw-4098a-predictor-f7bdb8d96 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:40:58Z" name: message-dumper-raw-4098a-predictor.18c45eddba908f3c namespace: kserve-ci-e2e-test resourceVersion: "17422" uid: 316dd851-a1ff-4968-ac4f-2e4c2e8e5d62 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:41:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-4098a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17413" uid: 27136dee-4c8e-4320-aa0a-cc6f6dc6aa80 kind: Event lastTimestamp: "2026-07-21T17:41:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:41:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:41:43Z" name: message-dumper-raw-4098a-predictor.18c45ee138fa179d namespace: kserve-ci-e2e-test resourceVersion: "17907" uid: 892463bd-5091-4bc4-9041-18e868690ebe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:41:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-4098a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17413" uid: 27136dee-4c8e-4320-aa0a-cc6f6dc6aa80 kind: Event lastTimestamp: "2026-07-21T17:41:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:41:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:41:43Z" name: message-dumper-raw-4098a-predictor.18c45ee1396eb435 namespace: kserve-ci-e2e-test resourceVersion: "17908" uid: b9104b47-e20a-45f4-88af-2cf1648335c1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-4098a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17614" uid: 27136dee-4c8e-4320-aa0a-cc6f6dc6aa80 kind: Event lastTimestamp: "2026-07-21T17:41:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:41:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:41:28Z" name: message-dumper-raw-4098a-predictor.18c45ee4b8ab67e2 namespace: kserve-ci-e2e-test resourceVersion: "17790" uid: e825b58f-8f1a-4d2f-b0d1-a1ffe3a16e42 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-4098a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17614" uid: 27136dee-4c8e-4320-aa0a-cc6f6dc6aa80 kind: Event lastTimestamp: "2026-07-21T17:41:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:41:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:41:28Z" name: message-dumper-raw-4098a-predictor.18c45ee4b92ca242 namespace: kserve-ci-e2e-test resourceVersion: "17792" uid: 46d345e4-c9d1-486e-a726-05da9e24a82f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-21T17:40:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-4098a namespace: kserve-ci-e2e-test resourceVersion: "17405" uid: 592edf9c-2170-492f-adf7-5caed906adb7 kind: Event lastTimestamp: "2026-07-21T17:41:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:41:09Z" name: message-dumper-raw-4098a.18c45eddba8ff68d namespace: kserve-ci-e2e-test resourceVersion: "17572" uid: 49b7d1c4-5720-460b-b832-e3f389b4838b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:40:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-4098a namespace: kserve-ci-e2e-test resourceVersion: "17424" uid: 592edf9c-2170-492f-adf7-5caed906adb7 kind: Event lastTimestamp: "2026-07-21T17:40:58Z" message: 'Failed to update status for InferenceService "message-dumper-raw-4098a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-4098a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:40:58Z" name: message-dumper-raw-4098a.18c45eddbeaac6ff namespace: kserve-ci-e2e-test resourceVersion: "17446" uid: c28b03a1-bab6-41a2-872e-37585f7689d6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:41:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-4098a namespace: kserve-ci-e2e-test resourceVersion: "17570" uid: 592edf9c-2170-492f-adf7-5caed906adb7 kind: Event lastTimestamp: "2026-07-21T17:41:08Z" message: InferenceService [message-dumper-raw-4098a] is Ready metadata: creationTimestamp: "2026-07-21T17:41:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:41:08Z" name: message-dumper-raw-4098a.18c45ee01b975a04 namespace: kserve-ci-e2e-test resourceVersion: "17571" uid: 1a1f566c-cfaf-4812-b64a-717e986db9c9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:38:58.197379Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-69373-86554fb68c-9fhk8 namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: 655f1735-a076-4136-b378-dd9f2459f4bc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-69373-86554fb68c-9fhk8 to ip-10-0-129-47.ec2.internal metadata: creationTimestamp: "2026-07-21T17:38:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:38:58Z" name: model-chainer-raw-69373-86554fb68c-9fhk8.18c45ec1a8b05c93 namespace: kserve-ci-e2e-test resourceVersion: "16232" uid: 52fe7380-1335-4a3d-9e92-55587831e9bf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf7f8bd85-gq8p9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:58Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-69373-86554fb68c-9fhk8 namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: 655f1735-a076-4136-b378-dd9f2459f4bc kind: Event lastTimestamp: "2026-07-21T17:38:58Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:38:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:38:58Z" name: model-chainer-raw-69373-86554fb68c-9fhk8.18c45ec1c2102517 namespace: kserve-ci-e2e-test resourceVersion: "16247" uid: d1d78041-17c4-4b65-a391-aed9ab28626f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-69373} kind: Pod name: model-chainer-raw-69373-86554fb68c-9fhk8 namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: 655f1735-a076-4136-b378-dd9f2459f4bc kind: Event lastTimestamp: "2026-07-21T17:38:58Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:4c5b35bc47640cedc469a58dc4db3d0dafd2fc41a58ddd3f20a70ce44e68325e" metadata: creationTimestamp: "2026-07-21T17:38:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:58Z" name: model-chainer-raw-69373-86554fb68c-9fhk8.18c45ec1c376ff48 namespace: kserve-ci-e2e-test resourceVersion: "16249" uid: 2af8bc68-e8d8-47d0-bda9-fce7759802bf reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-47.ec2.internal source: component: kubelet host: ip-10-0-129-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-69373} kind: Pod name: model-chainer-raw-69373-86554fb68c-9fhk8 namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: 655f1735-a076-4136-b378-dd9f2459f4bc kind: Event lastTimestamp: "2026-07-21T17:39:01Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:4c5b35bc47640cedc469a58dc4db3d0dafd2fc41a58ddd3f20a70ce44e68325e" in 2.588s (2.588s including waiting). Image size: 218229850 bytes.' metadata: creationTimestamp: "2026-07-21T17:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:01Z" name: model-chainer-raw-69373-86554fb68c-9fhk8.18c45ec25dbfca3a namespace: kserve-ci-e2e-test resourceVersion: "16269" uid: b4775b8c-ddc7-48cf-9c81-abc054d9e77f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-47.ec2.internal source: component: kubelet host: ip-10-0-129-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-69373} kind: Pod name: model-chainer-raw-69373-86554fb68c-9fhk8 namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: 655f1735-a076-4136-b378-dd9f2459f4bc kind: Event lastTimestamp: "2026-07-21T17:39:01Z" message: 'Created container: model-chainer-raw-69373' metadata: creationTimestamp: "2026-07-21T17:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:01Z" name: model-chainer-raw-69373-86554fb68c-9fhk8.18c45ec260ad2fe3 namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: d58adac6-1d80-4c6e-a961-d52af335c9d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-47.ec2.internal source: component: kubelet host: ip-10-0-129-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-69373} kind: Pod name: model-chainer-raw-69373-86554fb68c-9fhk8 namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: 655f1735-a076-4136-b378-dd9f2459f4bc kind: Event lastTimestamp: "2026-07-21T17:39:01Z" message: Started container model-chainer-raw-69373 metadata: creationTimestamp: "2026-07-21T17:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:01Z" name: model-chainer-raw-69373-86554fb68c-9fhk8.18c45ec261225158 namespace: kserve-ci-e2e-test resourceVersion: "16271" uid: 9dc6cf79-bde3-42df-9208-44a02421f01f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-47.ec2.internal source: component: kubelet host: ip-10-0-129-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-69373} kind: Pod name: model-chainer-raw-69373-86554fb68c-9fhk8 namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: 655f1735-a076-4136-b378-dd9f2459f4bc kind: Event lastTimestamp: "2026-07-21T17:39:08Z" message: Stopping container model-chainer-raw-69373 metadata: creationTimestamp: "2026-07-21T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:08Z" name: model-chainer-raw-69373-86554fb68c-9fhk8.18c45ec40015caac namespace: kserve-ci-e2e-test resourceVersion: "16345" uid: 46b5d980-92a6-42b5-b18c-71684922d7ba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-47.ec2.internal source: component: kubelet host: ip-10-0-129-47.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-69373} kind: Pod name: model-chainer-raw-69373-86554fb68c-9fhk8 namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: 655f1735-a076-4136-b378-dd9f2459f4bc kind: Event lastTimestamp: "2026-07-21T17:39:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-21T17:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:37Z" name: model-chainer-raw-69373-86554fb68c-9fhk8.18c45ec50c57f465 namespace: kserve-ci-e2e-test resourceVersion: "16734" uid: 15b82e7d-9e45-4427-8342-0e0685e45ada reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-47.ec2.internal source: component: kubelet host: ip-10-0-129-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-69373-86554fb68c namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 05aee756-db63-4e17-b09e-ecd07202c0a0 kind: Event lastTimestamp: "2026-07-21T17:38:58Z" message: 'Created pod: model-chainer-raw-69373-86554fb68c-9fhk8' metadata: creationTimestamp: "2026-07-21T17:38:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:58Z" name: model-chainer-raw-69373-86554fb68c.18c45ec1a83a4d5c namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: 1c3308d7-0c9f-40d2-9c2b-e3a6eb266ebb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-69373 namespace: kserve-ci-e2e-test resourceVersion: "16222" uid: 06610fc1-faec-4c03-a681-2568dd4749b3 kind: Event lastTimestamp: "2026-07-21T17:38:58Z" message: Scaled up replica set model-chainer-raw-69373-86554fb68c from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:38:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:58Z" name: model-chainer-raw-69373.18c45ec1a74880ae namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 0c0edeef-170a-4f00-96aa-2d42ab8efd56 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:58Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-69373 namespace: kserve-ci-e2e-test resourceVersion: "16239" uid: 7ded4206-0ef0-40a9-b59d-db8a9658c539 kind: Event lastTimestamp: "2026-07-21T17:38:58Z" message: InferenceGraph [model-chainer-raw-69373] is Ready metadata: creationTimestamp: "2026-07-21T17:38:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:38:58Z" name: model-chainer-raw-69373.18c45ec1ad16b1e6 namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: aa49d907-75f0-4165-b017-c90f8905dbac reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:40:48.523352Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-3ad06-7f4c64b959-5w2lz namespace: kserve-ci-e2e-test resourceVersion: "17282" uid: e02b69de-1823-4071-bc7f-9eae15f974a6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-3ad06-7f4c64b959-5w2lz to ip-10-0-129-47.ec2.internal metadata: creationTimestamp: "2026-07-21T17:40:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:40:48Z" name: model-chainer-raw-hpa-3ad06-7f4c64b959-5w2lz.18c45edb58a10f8c namespace: kserve-ci-e2e-test resourceVersion: "17289" uid: b496eb3a-9f17-47ea-bc69-b4b0e9548528 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf7f8bd85-gq8p9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:40:48Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-3ad06-7f4c64b959-5w2lz namespace: kserve-ci-e2e-test resourceVersion: "17292" uid: e02b69de-1823-4071-bc7f-9eae15f974a6 kind: Event lastTimestamp: "2026-07-21T17:40:48Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:40:48Z" name: model-chainer-raw-hpa-3ad06-7f4c64b959-5w2lz.18c45edb71fff3ab namespace: kserve-ci-e2e-test resourceVersion: "17303" uid: 2dfed138-9d12-4320-a2f7-260f7ce3033a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:40:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3ad06} kind: Pod name: model-chainer-raw-hpa-3ad06-7f4c64b959-5w2lz namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: e02b69de-1823-4071-bc7f-9eae15f974a6 kind: Event lastTimestamp: "2026-07-21T17:40:48Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4c5b35bc47640cedc469a58dc4db3d0dafd2fc41a58ddd3f20a70ce44e68325e" already present on machine metadata: creationTimestamp: "2026-07-21T17:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:40:48Z" name: model-chainer-raw-hpa-3ad06-7f4c64b959-5w2lz.18c45edb7347eac3 namespace: kserve-ci-e2e-test resourceVersion: "17305" uid: 6aa20733-bc08-4627-8ce9-9fce4d78b4fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-47.ec2.internal source: component: kubelet host: ip-10-0-129-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:40:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3ad06} kind: Pod name: model-chainer-raw-hpa-3ad06-7f4c64b959-5w2lz namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: e02b69de-1823-4071-bc7f-9eae15f974a6 kind: Event lastTimestamp: "2026-07-21T17:40:49Z" message: 'Created container: model-chainer-raw-hpa-3ad06' metadata: creationTimestamp: "2026-07-21T17:40:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:40:49Z" name: model-chainer-raw-hpa-3ad06-7f4c64b959-5w2lz.18c45edb767e2c9d namespace: kserve-ci-e2e-test resourceVersion: "17306" uid: 51d08ac9-89c6-4c61-8c57-e192adf4b4bb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-47.ec2.internal source: component: kubelet host: ip-10-0-129-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:40:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3ad06} kind: Pod name: model-chainer-raw-hpa-3ad06-7f4c64b959-5w2lz namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: e02b69de-1823-4071-bc7f-9eae15f974a6 kind: Event lastTimestamp: "2026-07-21T17:40:49Z" message: Started container model-chainer-raw-hpa-3ad06 metadata: creationTimestamp: "2026-07-21T17:40:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:40:49Z" name: model-chainer-raw-hpa-3ad06-7f4c64b959-5w2lz.18c45edb76ef6b0b namespace: kserve-ci-e2e-test resourceVersion: "17307" uid: 966cdace-ffb0-4213-981d-a8e0f612915a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-47.ec2.internal source: component: kubelet host: ip-10-0-129-47.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3ad06} kind: Pod name: model-chainer-raw-hpa-3ad06-7f4c64b959-5w2lz namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: e02b69de-1823-4071-bc7f-9eae15f974a6 kind: Event lastTimestamp: "2026-07-21T17:40:58Z" message: Stopping container model-chainer-raw-hpa-3ad06 metadata: creationTimestamp: "2026-07-21T17:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:40:58Z" name: model-chainer-raw-hpa-3ad06-7f4c64b959-5w2lz.18c45eddb1bde774 namespace: kserve-ci-e2e-test resourceVersion: "17399" uid: 2ec96edd-dc2d-4870-81fe-8fb75dfe81ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-47.ec2.internal source: component: kubelet host: ip-10-0-129-47.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:41:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3ad06} kind: Pod name: model-chainer-raw-hpa-3ad06-7f4c64b959-5w2lz namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: e02b69de-1823-4071-bc7f-9eae15f974a6 kind: Event lastTimestamp: "2026-07-21T17:41:25Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-21T17:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:41:25Z" name: model-chainer-raw-hpa-3ad06-7f4c64b959-5w2lz.18c45ede0daeb07a namespace: kserve-ci-e2e-test resourceVersion: "17762" uid: 3db843eb-a1a2-4f5d-b319-ea23dfe88f0a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-47.ec2.internal source: component: kubelet host: ip-10-0-129-47.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:40:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-3ad06-7f4c64b959 namespace: kserve-ci-e2e-test resourceVersion: "17279" uid: d0c084d3-d798-4c3a-b426-8da9d11804a9 kind: Event lastTimestamp: "2026-07-21T17:40:48Z" message: 'Created pod: model-chainer-raw-hpa-3ad06-7f4c64b959-5w2lz' metadata: creationTimestamp: "2026-07-21T17:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:40:48Z" name: model-chainer-raw-hpa-3ad06-7f4c64b959.18c45edb5834dcfe namespace: kserve-ci-e2e-test resourceVersion: "17284" uid: ab85c494-c365-4273-9f30-c5d28c649070 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:40:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-3ad06 namespace: kserve-ci-e2e-test resourceVersion: "17277" uid: dec3b909-39d3-4189-859f-fb91ad5e8095 kind: Event lastTimestamp: "2026-07-21T17:40:48Z" message: Scaled up replica set model-chainer-raw-hpa-3ad06-7f4c64b959 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:40:48Z" name: model-chainer-raw-hpa-3ad06.18c45edb5743418e namespace: kserve-ci-e2e-test resourceVersion: "17280" uid: 3f724f51-541c-4c0e-a50f-fa85cf26172b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:40:48Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-3ad06 namespace: kserve-ci-e2e-test resourceVersion: "17294" uid: 2ebe60b1-5e82-46a7-a13b-ae36391a4d92 kind: Event lastTimestamp: "2026-07-21T17:40:48Z" message: InferenceGraph [model-chainer-raw-hpa-3ad06] is Ready metadata: creationTimestamp: "2026-07-21T17:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:40:48Z" name: model-chainer-raw-hpa-3ad06.18c45edb5c30e26c namespace: kserve-ci-e2e-test resourceVersion: "17296" uid: 7e68f70e-16ad-403a-98cd-d825ecc1b82d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:48:01.206974Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp namespace: kserve-ci-e2e-test resourceVersion: "20986" uid: 5cd55f48-6ecf-458e-9de5-71a8480d9f71 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-9487a-predictor-7ff69748bd-pfwzp to ip-10-0-136-31.ec2.internal metadata: creationTimestamp: "2026-07-21T17:48:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:48:01Z" name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp.18c45f4016953b5f namespace: kserve-ci-e2e-test resourceVersion: "20992" uid: 8f36d7be-9c0f-4eff-906e-8326e65db707 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf7f8bd85-gq8p9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:01Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp namespace: kserve-ci-e2e-test resourceVersion: "20996" uid: 5cd55f48-6ecf-458e-9de5-71a8480d9f71 kind: Event lastTimestamp: "2026-07-21T17:48:01Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:48:01Z" name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp.18c45f4031a7275a namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: 2a1bcdf2-e4cf-4d53-b8e4-51349d7c17a5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 5cd55f48-6ecf-458e-9de5-71a8480d9f71 kind: Event lastTimestamp: "2026-07-21T17:48:01Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e12dde312cd424230e1ef8053d8890ee8a23543039767c0a577fe647c927b0a" already present on machine metadata: creationTimestamp: "2026-07-21T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:01Z" name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp.18c45f403362ae06 namespace: kserve-ci-e2e-test resourceVersion: "21006" uid: e71ec16f-206c-4329-b26c-0047fb5ee19c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 5cd55f48-6ecf-458e-9de5-71a8480d9f71 kind: Event lastTimestamp: "2026-07-21T17:48:01Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-21T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:01Z" name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp.18c45f40362f1bb6 namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: c44d20bf-b7be-4471-88ae-6fd601759c6f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 5cd55f48-6ecf-458e-9de5-71a8480d9f71 kind: Event lastTimestamp: "2026-07-21T17:48:01Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-21T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:01Z" name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp.18c45f40369f5c1b namespace: kserve-ci-e2e-test resourceVersion: "21008" uid: 5e65209f-4a6c-4d1e-be51-0b2d0ac05319 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 5cd55f48-6ecf-458e-9de5-71a8480d9f71 kind: Event lastTimestamp: "2026-07-21T17:48:05Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:05Z" name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp.18c45f4122ac7dbf namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: 052500ca-bfd1-4dfb-83b3-4926a57e7a5d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 5cd55f48-6ecf-458e-9de5-71a8480d9f71 kind: Event lastTimestamp: "2026-07-21T17:48:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:05Z" name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp.18c45f4125aabc00 namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: e45a6307-9be4-4329-be58-4341d9433b5d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 5cd55f48-6ecf-458e-9de5-71a8480d9f71 kind: Event lastTimestamp: "2026-07-21T17:48:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:05Z" name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp.18c45f41261d3d1d namespace: kserve-ci-e2e-test resourceVersion: "21048" uid: 31c85130-a679-43c1-b44f-1503b7460c0a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 5cd55f48-6ecf-458e-9de5-71a8480d9f71 kind: Event lastTimestamp: "2026-07-21T17:48:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:05Z" name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp.18c45f4126265d4d namespace: kserve-ci-e2e-test resourceVersion: "21049" uid: 0b5ece95-10ac-4f7b-b0a9-6b12e00e2cad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 5cd55f48-6ecf-458e-9de5-71a8480d9f71 kind: Event lastTimestamp: "2026-07-21T17:48:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:05Z" name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp.18c45f4128b28457 namespace: kserve-ci-e2e-test resourceVersion: "21052" uid: a5c5e618-b119-44fd-9d72-d2bbbd471483 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 5cd55f48-6ecf-458e-9de5-71a8480d9f71 kind: Event lastTimestamp: "2026-07-21T17:48:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:05Z" name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp.18c45f41291f5043 namespace: kserve-ci-e2e-test resourceVersion: "21053" uid: 35639ccf-dd05-4f40-9a98-1e0300d26382 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-21T17:48:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 5cd55f48-6ecf-458e-9de5-71a8480d9f71 kind: Event lastTimestamp: "2026-07-21T17:49:12Z" message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:48:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:12Z" name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp.18c45f415eba99ea namespace: kserve-ci-e2e-test resourceVersion: "21536" uid: cff5ef3c-2ce0-4329-a5f8-75aa2c301f8f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 5cd55f48-6ecf-458e-9de5-71a8480d9f71 kind: Event lastTimestamp: "2026-07-21T17:49:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:49:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:21Z" name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp.18c45f52bd089eef namespace: kserve-ci-e2e-test resourceVersion: "21617" uid: cb54edcc-70aa-4b9d-bdab-9336b87a4f11 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 5cd55f48-6ecf-458e-9de5-71a8480d9f71 kind: Event lastTimestamp: "2026-07-21T17:49:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:49:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:21Z" name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp.18c45f52bd09522d namespace: kserve-ci-e2e-test resourceVersion: "21618" uid: 3a74cbd1-38d8-40ce-8f54-2e6355e3f22a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 5cd55f48-6ecf-458e-9de5-71a8480d9f71 kind: Event lastTimestamp: "2026-07-21T17:49:22Z" message: 'Readiness probe failed: Get "https://10.133.0.41:8643/healthz": dial tcp 10.133.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:49:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:22Z" name: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp.18c45f5310e67c0a namespace: kserve-ci-e2e-test resourceVersion: "21630" uid: 42e71013-48b1-461c-bed0-493b6631df96 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-9487a-predictor-7ff69748bd namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: de8fe500-b260-4fa6-b34e-e551c70b8ea4 kind: Event lastTimestamp: "2026-07-21T17:48:01Z" message: 'Created pod: raw-sklearn-9487a-predictor-7ff69748bd-pfwzp' metadata: creationTimestamp: "2026-07-21T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:01Z" name: raw-sklearn-9487a-predictor-7ff69748bd.18c45f40162c266c namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: b16ea1ff-f0d2-438f-857f-060132ae6d30 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-9487a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20973" uid: 020bec0b-266d-4c18-b071-03b11ee59443 kind: Event lastTimestamp: "2026-07-21T17:48:01Z" message: Scaled up replica set raw-sklearn-9487a-predictor-7ff69748bd from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:01Z" name: raw-sklearn-9487a-predictor.18c45f4013082303 namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: 725221c8-222a-463d-9f06-9ef190b12a67 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:48:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-9487a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20974" uid: 0b594c9f-7307-4dc3-99cf-ca3c58898d7c kind: Event lastTimestamp: "2026-07-21T17:48:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:46Z" name: raw-sklearn-9487a-predictor.18c45f4391925e40 namespace: kserve-ci-e2e-test resourceVersion: "21348" uid: fec7d8eb-dedd-4d88-a412-baa27d80468d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:48:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-9487a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20974" uid: 0b594c9f-7307-4dc3-99cf-ca3c58898d7c kind: Event lastTimestamp: "2026-07-21T17:48:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:46Z" name: raw-sklearn-9487a-predictor.18c45f439213cf08 namespace: kserve-ci-e2e-test resourceVersion: "21349" uid: 6cd52d0a-a6ac-4c37-909f-2f56d99a0da3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:49:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-9487a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: 0b594c9f-7307-4dc3-99cf-ca3c58898d7c kind: Event lastTimestamp: "2026-07-21T17:49:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:16Z" name: raw-sklearn-9487a-predictor.18c45f4e1050bea0 namespace: kserve-ci-e2e-test resourceVersion: "21557" uid: 741677db-928e-46a0-ab39-62bd3b64e9e3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:49:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-9487a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: 0b594c9f-7307-4dc3-99cf-ca3c58898d7c kind: Event lastTimestamp: "2026-07-21T17:49:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:16Z" name: raw-sklearn-9487a-predictor.18c45f4e10a0b743 namespace: kserve-ci-e2e-test resourceVersion: "21558" uid: 08211476-c155-41a7-a38f-e6e83dc495f1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-21T17:48:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-9487a namespace: kserve-ci-e2e-test resourceVersion: "20966" uid: 233faef1-1f90-4775-b39f-7a524241f906 kind: Event lastTimestamp: "2026-07-21T17:49:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:49:19Z" name: raw-sklearn-9487a.18c45f401309bd32 namespace: kserve-ci-e2e-test resourceVersion: "21585" uid: 2c5d7271-087a-4402-b3ec-02a35b95a8d8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-9487a namespace: kserve-ci-e2e-test resourceVersion: "20966" uid: 233faef1-1f90-4775-b39f-7a524241f906 kind: Event lastTimestamp: "2026-07-21T17:48:01Z" message: 'Failed to update status for InferenceService "raw-sklearn-9487a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-9487a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:48:01Z" name: raw-sklearn-9487a.18c45f4013aa44e7 namespace: kserve-ci-e2e-test resourceVersion: "20980" uid: ef201fbd-d95e-4211-9e3d-5742a09627c6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-9487a namespace: kserve-ci-e2e-test resourceVersion: "21583" uid: 233faef1-1f90-4775-b39f-7a524241f906 kind: Event lastTimestamp: "2026-07-21T17:49:19Z" message: InferenceService [raw-sklearn-9487a] is Ready metadata: creationTimestamp: "2026-07-21T17:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:49:19Z" name: raw-sklearn-9487a.18c45f5238ae55f7 namespace: kserve-ci-e2e-test resourceVersion: "21584" uid: af2fbb1d-2936-47a2-95dc-d2c03a250af4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:49:26.588356Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx namespace: kserve-ci-e2e-test resourceVersion: "21683" uid: 67986832-04c1-48ba-b359-f4a1474e3c53 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx to ip-10-0-136-31.ec2.internal metadata: creationTimestamp: "2026-07-21T17:49:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:49:26Z" name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx.18c45f53f7b5b22b namespace: kserve-ci-e2e-test resourceVersion: "21689" uid: 8e37d82d-424a-4fbc-add8-ba96b2afe73b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bf7f8bd85-gq8p9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:27Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx namespace: kserve-ci-e2e-test resourceVersion: "21693" uid: 67986832-04c1-48ba-b359-f4a1474e3c53 kind: Event lastTimestamp: "2026-07-21T17:49:27Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:49:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:49:27Z" name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx.18c45f5411412567 namespace: kserve-ci-e2e-test resourceVersion: "21701" uid: fd899d74-d283-4021-81c2-bc103c19f02f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 67986832-04c1-48ba-b359-f4a1474e3c53 kind: Event lastTimestamp: "2026-07-21T17:49:27Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e12dde312cd424230e1ef8053d8890ee8a23543039767c0a577fe647c927b0a" already present on machine metadata: creationTimestamp: "2026-07-21T17:49:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:27Z" name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx.18c45f5412d86bfb namespace: kserve-ci-e2e-test resourceVersion: "21703" uid: 6c820961-ca6a-492d-960b-2bc874f00aad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 67986832-04c1-48ba-b359-f4a1474e3c53 kind: Event lastTimestamp: "2026-07-21T17:49:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-21T17:49:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:27Z" name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx.18c45f5415ca0960 namespace: kserve-ci-e2e-test resourceVersion: "21704" uid: 96a9750f-a36e-4330-8e2d-e35207752491 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 67986832-04c1-48ba-b359-f4a1474e3c53 kind: Event lastTimestamp: "2026-07-21T17:49:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-21T17:49:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:27Z" name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx.18c45f54163d6d73 namespace: kserve-ci-e2e-test resourceVersion: "21705" uid: 648268e5-f254-4001-968f-24047f5634be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 67986832-04c1-48ba-b359-f4a1474e3c53 kind: Event lastTimestamp: "2026-07-21T17:49:31Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:31Z" name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx.18c45f550281d6c2 namespace: kserve-ci-e2e-test resourceVersion: "21740" uid: 4ef3c93e-6497-41ae-ad9b-afbff2fa77f7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 67986832-04c1-48ba-b359-f4a1474e3c53 kind: Event lastTimestamp: "2026-07-21T17:49:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:31Z" name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx.18c45f5505d8abc5 namespace: kserve-ci-e2e-test resourceVersion: "21743" uid: e5c6837b-cf3c-4dca-95b5-d661f2dcd401 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 67986832-04c1-48ba-b359-f4a1474e3c53 kind: Event lastTimestamp: "2026-07-21T17:49:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:31Z" name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx.18c45f55069ce3db namespace: kserve-ci-e2e-test resourceVersion: "21745" uid: 46e1cf78-0ae0-409d-99a0-c46379ae5e86 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 67986832-04c1-48ba-b359-f4a1474e3c53 kind: Event lastTimestamp: "2026-07-21T17:49:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:31Z" name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx.18c45f5506a9eb44 namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 7a91ae08-54cc-4cef-9284-952b0be695c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 67986832-04c1-48ba-b359-f4a1474e3c53 kind: Event lastTimestamp: "2026-07-21T17:49:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:31Z" name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx.18c45f5509a93149 namespace: kserve-ci-e2e-test resourceVersion: "21749" uid: d8ce4193-c9a3-44ab-8d3a-11b5d8fc1562 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 67986832-04c1-48ba-b359-f4a1474e3c53 kind: Event lastTimestamp: "2026-07-21T17:49:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:31Z" name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx.18c45f550a19f325 namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: 0386210d-7323-4a1b-a763-d3839a79bfb1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-21T17:49:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 67986832-04c1-48ba-b359-f4a1474e3c53 kind: Event lastTimestamp: "2026-07-21T17:50:59Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:50:59Z" name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx.18c45f557a80f656 namespace: kserve-ci-e2e-test resourceVersion: "22404" uid: b62f98c9-687f-4d45-bd89-7bd458105d85 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:50:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 67986832-04c1-48ba-b359-f4a1474e3c53 kind: Event lastTimestamp: "2026-07-21T17:50:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:50:56Z" name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx.18c45f68fdc82a2d namespace: kserve-ci-e2e-test resourceVersion: "22387" uid: de71db83-9eb2-4d66-bb80-74e48b8e113d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:50:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 67986832-04c1-48ba-b359-f4a1474e3c53 kind: Event lastTimestamp: "2026-07-21T17:50:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:50:56Z" name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx.18c45f68fdc8b494 namespace: kserve-ci-e2e-test resourceVersion: "22388" uid: fcd10d68-9d3f-4ebf-b12d-85b30591ac49 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:50:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 67986832-04c1-48ba-b359-f4a1474e3c53 kind: Event lastTimestamp: "2026-07-21T17:50:59Z" message: 'Readiness probe failed: Get "https://10.133.0.42:8643/healthz": dial tcp 10.133.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:50:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:50:59Z" name: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx.18c45f6980c545b2 namespace: kserve-ci-e2e-test resourceVersion: "22403" uid: fd57f90e-47cc-4e2e-89b1-b59a702580ba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-31.ec2.internal source: component: kubelet host: ip-10-0-136-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-480dc-predictor-59b8d4c96 namespace: kserve-ci-e2e-test resourceVersion: "21674" uid: 0aa5e083-60dd-41f2-b2db-6a434f93d2ed kind: Event lastTimestamp: "2026-07-21T17:49:26Z" message: 'Created pod: raw-sklearn-runtime-480dc-predictor-59b8d4c96-f97kx' metadata: creationTimestamp: "2026-07-21T17:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:26Z" name: raw-sklearn-runtime-480dc-predictor-59b8d4c96.18c45f53f738f31a namespace: kserve-ci-e2e-test resourceVersion: "21686" uid: 310164e5-62d1-40a1-ac8d-fd556c007f61 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-480dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: beba90d9-f365-40d5-9a13-112506440961 kind: Event lastTimestamp: "2026-07-21T17:49:26Z" message: Scaled up replica set raw-sklearn-runtime-480dc-predictor-59b8d4c96 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:26Z" name: raw-sklearn-runtime-480dc-predictor.18c45f53f11a47c2 namespace: kserve-ci-e2e-test resourceVersion: "21676" uid: be474f02-2307-4491-9cda-ef018d1dbb15 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:49:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-480dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21673" uid: f4f59208-3ad7-41dd-9498-a23bacee09f2 kind: Event lastTimestamp: "2026-07-21T17:50:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:50:11Z" name: raw-sklearn-runtime-480dc-predictor.18c45f577003b5c6 namespace: kserve-ci-e2e-test resourceVersion: "22030" uid: e1942e4a-d155-45a1-ad8a-6eb86cc29583 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:49:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-480dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21673" uid: f4f59208-3ad7-41dd-9498-a23bacee09f2 kind: Event lastTimestamp: "2026-07-21T17:50:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:50:11Z" name: raw-sklearn-runtime-480dc-predictor.18c45f57706f239b namespace: kserve-ci-e2e-test resourceVersion: "22033" uid: e2f8c2f8-683a-467d-8699-1e236d8df785 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-21T17:49:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-480dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21825" uid: f4f59208-3ad7-41dd-9498-a23bacee09f2 kind: Event lastTimestamp: "2026-07-21T17:50:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:50:56Z" name: raw-sklearn-runtime-480dc-predictor.18c45f5aef9a2adc namespace: kserve-ci-e2e-test resourceVersion: "22368" uid: 52d719a9-e67f-4990-b385-9f428416e789 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-21T17:49:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-480dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21825" uid: f4f59208-3ad7-41dd-9498-a23bacee09f2 kind: Event lastTimestamp: "2026-07-21T17:50:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:50:56Z" name: raw-sklearn-runtime-480dc-predictor.18c45f5af008e1e0 namespace: kserve-ci-e2e-test resourceVersion: "22370" uid: 928fc659-9134-4a50-92d6-1e84c1f7a249 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-21T17:49:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-480dc namespace: kserve-ci-e2e-test resourceVersion: "21664" uid: a20e7345-28b1-429b-9502-b1f4e3f3778c kind: Event lastTimestamp: "2026-07-21T17:50:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:50:49Z" name: raw-sklearn-runtime-480dc.18c45f53f109dd7f namespace: kserve-ci-e2e-test resourceVersion: "22312" uid: 945bcf06-4188-409e-8532-c542d7c0f162 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-480dc namespace: kserve-ci-e2e-test resourceVersion: "21664" uid: a20e7345-28b1-429b-9502-b1f4e3f3778c kind: Event lastTimestamp: "2026-07-21T17:49:26Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-480dc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-480dc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:49:26Z" name: raw-sklearn-runtime-480dc.18c45f53f2203e52 namespace: kserve-ci-e2e-test resourceVersion: "21678" uid: 4af493cb-3bac-4f02-86fe-3cdd831858cd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:50:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-480dc namespace: kserve-ci-e2e-test resourceVersion: "22310" uid: a20e7345-28b1-429b-9502-b1f4e3f3778c kind: Event lastTimestamp: "2026-07-21T17:50:49Z" message: InferenceService [raw-sklearn-runtime-480dc] is Ready metadata: creationTimestamp: "2026-07-21T17:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:50:49Z" name: raw-sklearn-runtime-480dc.18c45f6732c3fcd9 namespace: kserve-ci-e2e-test resourceVersion: "22311" uid: 50fee42f-ecd4-4ee0-b4ed-732285ef3ddf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22640"