--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:56:56.246906Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-22eec6-predictor-584698c74b-2bmvv namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: ca641694-2844-4e99-99f2-5c4f75d3eaec kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-22eec6-predictor-584698c74b-2bmvv to ip-10-0-132-218.ec2.internal metadata: creationTimestamp: "2026-07-16T17:56:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:56:56Z" name: isvc-init-fail-22eec6-predictor-584698c74b-2bmvv.18c2d6d5d2e7a760 namespace: kserve-ci-e2e-test resourceVersion: "20696" uid: 0cee9ba1-924d-4f43-8a87-bb1c64a54cf7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-699544b6dc-lkd89 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-22eec6-predictor-584698c74b-2bmvv namespace: kserve-ci-e2e-test resourceVersion: "20692" uid: ca641694-2844-4e99-99f2-5c4f75d3eaec kind: Event lastTimestamp: "2026-07-16T17:56:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-init-fail-22eec6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T17:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:56Z" name: isvc-init-fail-22eec6-predictor-584698c74b-2bmvv.18c2d6d5dfe3a06a namespace: kserve-ci-e2e-test resourceVersion: "20707" uid: 51672ad6-50b0-4740-b82e-31fa56d1b6a0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-22eec6-predictor-584698c74b-2bmvv namespace: kserve-ci-e2e-test resourceVersion: "20701" uid: ca641694-2844-4e99-99f2-5c4f75d3eaec kind: Event lastTimestamp: "2026-07-16T17:56:57Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:56:57Z" name: isvc-init-fail-22eec6-predictor-584698c74b-2bmvv.18c2d6d610d15db5 namespace: kserve-ci-e2e-test resourceVersion: "20716" uid: 81b6d27a-fae8-4bb7-9a86-5ecbf2ef984b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-22eec6-predictor-584698c74b-2bmvv namespace: kserve-ci-e2e-test resourceVersion: "20692" uid: ca641694-2844-4e99-99f2-5c4f75d3eaec kind: Event lastTimestamp: "2026-07-16T17:57:01Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:86207d1cba5aecf8e7b5bca617ed35061e54e289c7778b383f47cc8d53214f1a" already present on machine metadata: creationTimestamp: "2026-07-16T17:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:01Z" name: isvc-init-fail-22eec6-predictor-584698c74b-2bmvv.18c2d6d612593685 namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: 82c4352f-d331-4566-839f-ee6b55ad53c0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-22eec6-predictor-584698c74b-2bmvv namespace: kserve-ci-e2e-test resourceVersion: "20692" uid: ca641694-2844-4e99-99f2-5c4f75d3eaec kind: Event lastTimestamp: "2026-07-16T17:57:01Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-16T17:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:01Z" name: isvc-init-fail-22eec6-predictor-584698c74b-2bmvv.18c2d6d615f1bf8a namespace: kserve-ci-e2e-test resourceVersion: "20757" uid: 6ed4050f-375f-4735-bb86-872a47b85900 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-22eec6-predictor-584698c74b-2bmvv namespace: kserve-ci-e2e-test resourceVersion: "20692" uid: ca641694-2844-4e99-99f2-5c4f75d3eaec kind: Event lastTimestamp: "2026-07-16T17:57:01Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-16T17:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:01Z" name: isvc-init-fail-22eec6-predictor-584698c74b-2bmvv.18c2d6d6166b75c1 namespace: kserve-ci-e2e-test resourceVersion: "20758" uid: ce48a4dd-d369-4aaf-a762-687195978458 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-22eec6-predictor-584698c74b-2bmvv namespace: kserve-ci-e2e-test resourceVersion: "20692" uid: ca641694-2844-4e99-99f2-5c4f75d3eaec kind: Event lastTimestamp: "2026-07-16T17:57:05Z" message: Back-off restarting failed container storage-initializer in pod isvc-init-fail-22eec6-predictor-584698c74b-2bmvv_kserve-ci-e2e-test(ca641694-2844-4e99-99f2-5c4f75d3eaec) metadata: creationTimestamp: "2026-07-16T17:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:05Z" name: isvc-init-fail-22eec6-predictor-584698c74b-2bmvv.18c2d6d8035d873a namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: 42e2d925-774c-4865-bf21-30ba5fb4c786 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-22eec6-predictor-584698c74b namespace: kserve-ci-e2e-test resourceVersion: "20681" uid: bdce44c6-fa9c-4e70-babf-c0348684ae1f kind: Event lastTimestamp: "2026-07-16T17:56:56Z" message: 'Created pod: isvc-init-fail-22eec6-predictor-584698c74b-2bmvv' metadata: creationTimestamp: "2026-07-16T17:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:56:56Z" name: isvc-init-fail-22eec6-predictor-584698c74b.18c2d6d5d24e6613 namespace: kserve-ci-e2e-test resourceVersion: "20691" uid: c9b8da79-f7ff-4b80-a52a-ca6702e30ee9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-22eec6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20679" uid: beb44fbe-dc80-40fe-a8b0-2806c8ea4d1f kind: Event lastTimestamp: "2026-07-16T17:56:56Z" message: Scaled up replica set isvc-init-fail-22eec6-predictor-584698c74b from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:56:56Z" name: isvc-init-fail-22eec6-predictor.18c2d6d5ced08988 namespace: kserve-ci-e2e-test resourceVersion: "20683" uid: 1f65708d-b7b8-496c-87e8-7d0968268193 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-16T17:56:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-22eec6 namespace: kserve-ci-e2e-test resourceVersion: "20672" uid: 8846dcfc-6e3c-4dd7-ab95-3260152f79a2 kind: Event lastTimestamp: "2026-07-16T17:57:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:57:05Z" name: isvc-init-fail-22eec6.18c2d6d5ceb3aae7 namespace: kserve-ci-e2e-test resourceVersion: "20797" uid: e3fc2383-a763-401a-ae73-b7fee5a61fe4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-22eec6 namespace: kserve-ci-e2e-test resourceVersion: "20672" uid: 8846dcfc-6e3c-4dd7-ab95-3260152f79a2 kind: Event lastTimestamp: "2026-07-16T17:56:56Z" message: 'Failed to update status for InferenceService "isvc-init-fail-22eec6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-22eec6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:56:56Z" name: isvc-init-fail-22eec6.18c2d6d5cf0d7b3f namespace: kserve-ci-e2e-test resourceVersion: "20686" uid: 2f1068d1-2bad-49e4-b839-9a231e0ddd6a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-22eec6 namespace: kserve-ci-e2e-test resourceVersion: "20672" uid: 8846dcfc-6e3c-4dd7-ab95-3260152f79a2 kind: Event lastTimestamp: "2026-07-16T17:56:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-22eec6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:56:56Z" name: isvc-init-fail-22eec6.18c2d6d5cf0dbca8 namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: 0fa36596-1067-4af9-9762-7b58bdeb85fa reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:50:31.947383Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: 4fc7b553-617d-4abe-9b21-aaa8054c93e5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-c3a17-predictor-d4997994d-8v68s to ip-10-0-132-218.ec2.internal metadata: creationTimestamp: "2026-07-16T17:50:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:50:31Z" name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s.18c2d67c58de46c5 namespace: kserve-ci-e2e-test resourceVersion: "17585" uid: ead9ac9d-446e-4c6f-91ca-031ddf257eca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-699544b6dc-lkd89 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:32Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s namespace: kserve-ci-e2e-test resourceVersion: "17589" uid: 4fc7b553-617d-4abe-9b21-aaa8054c93e5 kind: Event lastTimestamp: "2026-07-16T17:50:32Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:50:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:50:32Z" name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s.18c2d67c72c6f487 namespace: kserve-ci-e2e-test resourceVersion: "17593" uid: 61f7e573-51d6-4700-8838-60fca97a3f1b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 4fc7b553-617d-4abe-9b21-aaa8054c93e5 kind: Event lastTimestamp: "2026-07-16T17:50:32Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:86207d1cba5aecf8e7b5bca617ed35061e54e289c7778b383f47cc8d53214f1a" already present on machine metadata: creationTimestamp: "2026-07-16T17:50:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:32Z" name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s.18c2d67c747ae576 namespace: kserve-ci-e2e-test resourceVersion: "17595" uid: fa652f58-e6e7-40e6-9587-ce9d550dd4ee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 4fc7b553-617d-4abe-9b21-aaa8054c93e5 kind: Event lastTimestamp: "2026-07-16T17:50:32Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-16T17:50:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:32Z" name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s.18c2d67c77516be8 namespace: kserve-ci-e2e-test resourceVersion: "17598" uid: bcbc784e-8152-4010-b70b-fe34a890c1be reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 4fc7b553-617d-4abe-9b21-aaa8054c93e5 kind: Event lastTimestamp: "2026-07-16T17:50:32Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-16T17:50:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:32Z" name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s.18c2d67c77c97a3c namespace: kserve-ci-e2e-test resourceVersion: "17599" uid: 143876a7-78dc-40b4-b56b-12cdb6dcc2bd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 4fc7b553-617d-4abe-9b21-aaa8054c93e5 kind: Event lastTimestamp: "2026-07-16T17:50:36Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T17:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:36Z" name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s.18c2d67d82c3a38c namespace: kserve-ci-e2e-test resourceVersion: "17639" uid: dfe8bff1-e681-41cc-b226-71b83ded4951 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 4fc7b553-617d-4abe-9b21-aaa8054c93e5 kind: Event lastTimestamp: "2026-07-16T17:50:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:36Z" name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s.18c2d67d85e151c3 namespace: kserve-ci-e2e-test resourceVersion: "17642" uid: db0fab72-afb7-4e28-acf5-77647a09abf2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 4fc7b553-617d-4abe-9b21-aaa8054c93e5 kind: Event lastTimestamp: "2026-07-16T17:50:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:37Z" name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s.18c2d67d865dac44 namespace: kserve-ci-e2e-test resourceVersion: "17643" uid: a444b5a1-b443-4c13-90ab-149601783a86 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 4fc7b553-617d-4abe-9b21-aaa8054c93e5 kind: Event lastTimestamp: "2026-07-16T17:50:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:37Z" name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s.18c2d67d8669726f namespace: kserve-ci-e2e-test resourceVersion: "17644" uid: 55233476-4211-4383-9515-a829a5b13c82 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 4fc7b553-617d-4abe-9b21-aaa8054c93e5 kind: Event lastTimestamp: "2026-07-16T17:50:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:37Z" name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s.18c2d67d89a84010 namespace: kserve-ci-e2e-test resourceVersion: "17648" uid: 5950a683-944d-426d-8b83-722cd89a8ef7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 4fc7b553-617d-4abe-9b21-aaa8054c93e5 kind: Event lastTimestamp: "2026-07-16T17:50:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:37Z" name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s.18c2d67d8a20bbf0 namespace: kserve-ci-e2e-test resourceVersion: "17650" uid: 6f7c6887-4290-4240-b6c0-a403de4b821e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 4fc7b553-617d-4abe-9b21-aaa8054c93e5 kind: Event lastTimestamp: "2026-07-16T17:50:37Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:42ffb1a3fbaec62e57c71970d909bef17aa5b27605c06189276dc315c300eb9f" already present on machine metadata: creationTimestamp: "2026-07-16T17:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:37Z" name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s.18c2d67d8a2ba315 namespace: kserve-ci-e2e-test resourceVersion: "17651" uid: cb78ea4d-fd09-4210-85b4-813a9f0a2515 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 4fc7b553-617d-4abe-9b21-aaa8054c93e5 kind: Event lastTimestamp: "2026-07-16T17:50:37Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-16T17:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:37Z" name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s.18c2d67d8e63af94 namespace: kserve-ci-e2e-test resourceVersion: "17652" uid: 078d9742-9a36-4e2c-aca8-c0637061c1eb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 4fc7b553-617d-4abe-9b21-aaa8054c93e5 kind: Event lastTimestamp: "2026-07-16T17:50:37Z" message: Started container agent metadata: creationTimestamp: "2026-07-16T17:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:37Z" name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s.18c2d67d8ede6b30 namespace: kserve-ci-e2e-test resourceVersion: "17653" uid: 2ae615b7-c727-4ece-af6f-9b54fc3a0d72 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-16T17:50:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 4fc7b553-617d-4abe-9b21-aaa8054c93e5 kind: Event lastTimestamp: "2026-07-16T17:52:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:52:13Z" name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s.18c2d67dbeff1f9b namespace: kserve-ci-e2e-test resourceVersion: "18470" uid: 5b938bbc-a9b4-4719-a4fd-18a1b80e9d3d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-16T17:50:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 4fc7b553-617d-4abe-9b21-aaa8054c93e5 kind: Event lastTimestamp: "2026-07-16T17:52:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T17:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:52:23Z" name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s.18c2d67dbf0a8b63 namespace: kserve-ci-e2e-test resourceVersion: "18548" uid: 73dbd720-f2a7-48d2-a9d0-e94f6c3e715a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 4fc7b553-617d-4abe-9b21-aaa8054c93e5 kind: Event lastTimestamp: "2026-07-16T17:52:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:52:07Z" name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s.18c2d692867f97f2 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: 012229e4-112d-4276-91ba-e70c7c604f69 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 4fc7b553-617d-4abe-9b21-aaa8054c93e5 kind: Event lastTimestamp: "2026-07-16T17:52:07Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-16T17:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:52:07Z" name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s.18c2d6928680cdbf namespace: kserve-ci-e2e-test resourceVersion: "18355" uid: 3ea45a25-81d5-4dc7-a18d-e62c1cd957b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 4fc7b553-617d-4abe-9b21-aaa8054c93e5 kind: Event lastTimestamp: "2026-07-16T17:52:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:52:07Z" name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s.18c2d6928680cf00 namespace: kserve-ci-e2e-test resourceVersion: "18357" uid: d6a30887-6e61-4fa6-be7e-ee5f98cd2b5b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T17:52:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 4fc7b553-617d-4abe-9b21-aaa8054c93e5 kind: Event lastTimestamp: "2026-07-16T17:52:23Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:52:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:52:23Z" name: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s.18c2d692ef4b647a namespace: kserve-ci-e2e-test resourceVersion: "18546" uid: 9b61b15c-f114-4e7c-836f-ab406cff5d7d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-c3a17-predictor-d4997994d namespace: kserve-ci-e2e-test resourceVersion: "17568" uid: 8435bd21-d974-43e0-ada3-98cbcbc03db9 kind: Event lastTimestamp: "2026-07-16T17:50:31Z" message: 'Created pod: isvc-logger-raw-c3a17-predictor-d4997994d-8v68s' metadata: creationTimestamp: "2026-07-16T17:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:50:31Z" name: isvc-logger-raw-c3a17-predictor-d4997994d.18c2d67c5867050a namespace: kserve-ci-e2e-test resourceVersion: "17582" uid: 597573eb-1383-41c6-abaf-0fd76dc36c92 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-c3a17-predictor namespace: kserve-ci-e2e-test resourceVersion: "17566" uid: 4683dc1f-9835-4af2-89e7-f78db9b01996 kind: Event lastTimestamp: "2026-07-16T17:50:31Z" message: Scaled up replica set isvc-logger-raw-c3a17-predictor-d4997994d from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:50:31Z" name: isvc-logger-raw-c3a17-predictor.18c2d67c54013958 namespace: kserve-ci-e2e-test resourceVersion: "17570" uid: 0adfa8d2-9eef-4565-94d8-e2845bac024d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:50:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-c3a17-predictor namespace: kserve-ci-e2e-test resourceVersion: "17567" uid: 7138bd84-16dc-4151-848c-926ba475c9ea kind: Event lastTimestamp: "2026-07-16T17:51:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:51:01Z" name: isvc-logger-raw-c3a17-predictor.18c2d67fd2a1be6f namespace: kserve-ci-e2e-test resourceVersion: "17841" uid: 75d7338b-1e7b-43ab-9bae-95a050303462 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:50:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-c3a17-predictor namespace: kserve-ci-e2e-test resourceVersion: "17567" uid: 7138bd84-16dc-4151-848c-926ba475c9ea kind: Event lastTimestamp: "2026-07-16T17:51:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:51:01Z" name: isvc-logger-raw-c3a17-predictor.18c2d67fd329f7ef namespace: kserve-ci-e2e-test resourceVersion: "17842" uid: d71ea207-4b25-4ab4-b220-93765999e8f5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T17:51:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-c3a17-predictor namespace: kserve-ci-e2e-test resourceVersion: "17729" uid: 7138bd84-16dc-4151-848c-926ba475c9ea kind: Event lastTimestamp: "2026-07-16T17:52:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:51:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:52:01Z" name: isvc-logger-raw-c3a17-predictor.18c2d686d0eb35c8 namespace: kserve-ci-e2e-test resourceVersion: "18284" uid: 7daf2c93-5acb-446e-8b3c-5ce785a0ab1a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T17:51:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-c3a17-predictor namespace: kserve-ci-e2e-test resourceVersion: "17729" uid: 7138bd84-16dc-4151-848c-926ba475c9ea kind: Event lastTimestamp: "2026-07-16T17:52:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:51:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:52:01Z" name: isvc-logger-raw-c3a17-predictor.18c2d686d1551c96 namespace: kserve-ci-e2e-test resourceVersion: "18285" uid: d2476efd-2f4a-42a5-ba73-8312155524ea reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-16T17:50:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-c3a17 namespace: kserve-ci-e2e-test resourceVersion: "17559" uid: d3740df2-b102-4c0d-8c1d-8749bf62e5bd kind: Event lastTimestamp: "2026-07-16T17:51:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:51:54Z" name: isvc-logger-raw-c3a17.18c2d67c540d05cd namespace: kserve-ci-e2e-test resourceVersion: "18225" uid: 3631e965-10dc-4321-93cb-960162afa441 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-c3a17 namespace: kserve-ci-e2e-test resourceVersion: "17559" uid: d3740df2-b102-4c0d-8c1d-8749bf62e5bd kind: Event lastTimestamp: "2026-07-16T17:50:31Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-c3a17": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-c3a17": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:50:31Z" name: isvc-logger-raw-c3a17.18c2d67c5493ad1e namespace: kserve-ci-e2e-test resourceVersion: "17573" uid: 5b26e83e-ae46-42b1-983f-8e63b6718892 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:51:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-c3a17 namespace: kserve-ci-e2e-test resourceVersion: "18223" uid: d3740df2-b102-4c0d-8c1d-8749bf62e5bd kind: Event lastTimestamp: "2026-07-16T17:51:54Z" message: InferenceService [isvc-logger-raw-c3a17] is Ready metadata: creationTimestamp: "2026-07-16T17:51:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:51:54Z" name: isvc-logger-raw-c3a17.18c2d68f793518ba namespace: kserve-ci-e2e-test resourceVersion: "18224" uid: 6a661a53-0c13-49fb-9058-740fa1754935 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:55:02.737856Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-aae39d-predictor-d67696c6f-46tks namespace: kserve-ci-e2e-test resourceVersion: "19710" uid: f835a159-e668-421c-b478-2a7d9761aa01 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-aae39d-predictor-d67696c6f-46tks to ip-10-0-132-218.ec2.internal metadata: creationTimestamp: "2026-07-16T17:55:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:55:02Z" name: isvc-primary-aae39d-predictor-d67696c6f-46tks.18c2d6bb653cfefd namespace: kserve-ci-e2e-test resourceVersion: "19714" uid: ed32ac5a-aaf7-4d83-bc1a-13dc6c3c1ac8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-699544b6dc-lkd89 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:55:03Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-aae39d-predictor-d67696c6f-46tks namespace: kserve-ci-e2e-test resourceVersion: "19719" uid: f835a159-e668-421c-b478-2a7d9761aa01 kind: Event lastTimestamp: "2026-07-16T17:55:03Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:55:03Z" name: isvc-primary-aae39d-predictor-d67696c6f-46tks.18c2d6bb7f2f8e10 namespace: kserve-ci-e2e-test resourceVersion: "19728" uid: a10dbe49-e3fa-4514-9913-bc3d01cd1f8c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:55:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-aae39d-predictor-d67696c6f-46tks namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: f835a159-e668-421c-b478-2a7d9761aa01 kind: Event lastTimestamp: "2026-07-16T17:55:03Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:86207d1cba5aecf8e7b5bca617ed35061e54e289c7778b383f47cc8d53214f1a" already present on machine metadata: creationTimestamp: "2026-07-16T17:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:55:03Z" name: isvc-primary-aae39d-predictor-d67696c6f-46tks.18c2d6bb811e351a namespace: kserve-ci-e2e-test resourceVersion: "19730" uid: 04848899-1a88-44ba-92a4-b7bccfdf9b2b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:55:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-aae39d-predictor-d67696c6f-46tks namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: f835a159-e668-421c-b478-2a7d9761aa01 kind: Event lastTimestamp: "2026-07-16T17:55:03Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-16T17:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:55:03Z" name: isvc-primary-aae39d-predictor-d67696c6f-46tks.18c2d6bb83e0a7c4 namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 3a770b01-c242-48f0-aeaf-3415c9e709fa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:55:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-aae39d-predictor-d67696c6f-46tks namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: f835a159-e668-421c-b478-2a7d9761aa01 kind: Event lastTimestamp: "2026-07-16T17:55:03Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-16T17:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:55:03Z" name: isvc-primary-aae39d-predictor-d67696c6f-46tks.18c2d6bb8471c58b namespace: kserve-ci-e2e-test resourceVersion: "19732" uid: c536bbd2-7cfb-4832-a446-04c27f8a65ee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-aae39d-predictor-d67696c6f-46tks namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: f835a159-e668-421c-b478-2a7d9761aa01 kind: Event lastTimestamp: "2026-07-16T17:55:08Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:55:08Z" name: isvc-primary-aae39d-predictor-d67696c6f-46tks.18c2d6bca68ad023 namespace: kserve-ci-e2e-test resourceVersion: "19773" uid: df9aca89-c701-4e2f-9b64-f8b51862cfbb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-aae39d-predictor-d67696c6f-46tks namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: f835a159-e668-421c-b478-2a7d9761aa01 kind: Event lastTimestamp: "2026-07-16T17:55:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:55:08Z" name: isvc-primary-aae39d-predictor-d67696c6f-46tks.18c2d6bca9eba672 namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 624e8104-a695-488b-988d-839b840074da reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-aae39d-predictor-d67696c6f-46tks namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: f835a159-e668-421c-b478-2a7d9761aa01 kind: Event lastTimestamp: "2026-07-16T17:55:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:55:08Z" name: isvc-primary-aae39d-predictor-d67696c6f-46tks.18c2d6bcaa612cfa namespace: kserve-ci-e2e-test resourceVersion: "19779" uid: 9c712f7e-067e-4656-8f0c-6d9af3a999ef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-aae39d-predictor-d67696c6f-46tks namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: f835a159-e668-421c-b478-2a7d9761aa01 kind: Event lastTimestamp: "2026-07-16T17:55:08Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:55:08Z" name: isvc-primary-aae39d-predictor-d67696c6f-46tks.18c2d6bcaa6bc404 namespace: kserve-ci-e2e-test resourceVersion: "19780" uid: d9730075-62af-4a34-8cba-96a543d6cd64 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-aae39d-predictor-d67696c6f-46tks namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: f835a159-e668-421c-b478-2a7d9761aa01 kind: Event lastTimestamp: "2026-07-16T17:55:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:55:08Z" name: isvc-primary-aae39d-predictor-d67696c6f-46tks.18c2d6bcad13cf07 namespace: kserve-ci-e2e-test resourceVersion: "19783" uid: 4700bbbb-4f5e-4f43-8eb6-cd142b9148e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-aae39d-predictor-d67696c6f-46tks namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: f835a159-e668-421c-b478-2a7d9761aa01 kind: Event lastTimestamp: "2026-07-16T17:55:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:55:08Z" name: isvc-primary-aae39d-predictor-d67696c6f-46tks.18c2d6bcad88e6ac namespace: kserve-ci-e2e-test resourceVersion: "19784" uid: b59dc887-a8ed-418c-a905-c79e8d0c472f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-16T17:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-aae39d-predictor-d67696c6f-46tks namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: f835a159-e668-421c-b478-2a7d9761aa01 kind: Event lastTimestamp: "2026-07-16T17:56:55Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:55Z" name: isvc-primary-aae39d-predictor-d67696c6f-46tks.18c2d6bce2a69ed5 namespace: kserve-ci-e2e-test resourceVersion: "20655" uid: 53cb0775-ead4-4442-9823-e1123ccfc3a3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-aae39d-predictor-d67696c6f-46tks namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: f835a159-e668-421c-b478-2a7d9761aa01 kind: Event lastTimestamp: "2026-07-16T17:56:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:51Z" name: isvc-primary-aae39d-predictor-d67696c6f-46tks.18c2d6d49b385eb3 namespace: kserve-ci-e2e-test resourceVersion: "20606" uid: 8594e3f9-c0b4-4b23-aa0d-71d9df777ed1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-aae39d-predictor-d67696c6f-46tks namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: f835a159-e668-421c-b478-2a7d9761aa01 kind: Event lastTimestamp: "2026-07-16T17:56:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:51Z" name: isvc-primary-aae39d-predictor-d67696c6f-46tks.18c2d6d49b3980c4 namespace: kserve-ci-e2e-test resourceVersion: "20607" uid: 898e7f2b-3332-47fe-b2f3-fe687650d706 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-aae39d-predictor-d67696c6f-46tks namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: f835a159-e668-421c-b478-2a7d9761aa01 kind: Event lastTimestamp: "2026-07-16T17:56:55Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:56:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:55Z" name: isvc-primary-aae39d-predictor-d67696c6f-46tks.18c2d6d590ef2240 namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 2302fcfa-01d4-4be9-9ce9-7b4d67531b57 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:55:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-aae39d-predictor-d67696c6f namespace: kserve-ci-e2e-test resourceVersion: "19700" uid: 7f4439e9-1320-47a9-9799-540015e8c090 kind: Event lastTimestamp: "2026-07-16T17:55:02Z" message: 'Created pod: isvc-primary-aae39d-predictor-d67696c6f-46tks' metadata: creationTimestamp: "2026-07-16T17:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:55:02Z" name: isvc-primary-aae39d-predictor-d67696c6f.18c2d6bb64dfe0d5 namespace: kserve-ci-e2e-test resourceVersion: "19712" uid: 60414502-5577-4e20-9ed2-f57e40fc1f95 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:55:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-aae39d-predictor namespace: kserve-ci-e2e-test resourceVersion: "19698" uid: b4b47d0d-c162-40a8-bcbe-8a2f0e1ddc95 kind: Event lastTimestamp: "2026-07-16T17:55:02Z" message: Scaled up replica set isvc-primary-aae39d-predictor-d67696c6f from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:55:02Z" name: isvc-primary-aae39d-predictor.18c2d6bb5ec99362 namespace: kserve-ci-e2e-test resourceVersion: "19702" uid: 49cb5deb-2ccd-4048-b788-ea14e545738d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:55:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-aae39d-predictor namespace: kserve-ci-e2e-test resourceVersion: "19699" uid: 75aebf95-d3c7-43b8-a9c8-9351d4568927 kind: Event lastTimestamp: "2026-07-16T17:55:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:55:32Z" name: isvc-primary-aae39d-predictor.18c2d6bedd5c6c6f namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: 961596eb-6464-4b17-a4a6-206b27d78e7a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:55:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-aae39d-predictor namespace: kserve-ci-e2e-test resourceVersion: "19699" uid: 75aebf95-d3c7-43b8-a9c8-9351d4568927 kind: Event lastTimestamp: "2026-07-16T17:55:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:55:32Z" name: isvc-primary-aae39d-predictor.18c2d6beddc4e4fb namespace: kserve-ci-e2e-test resourceVersion: "19961" uid: 2afa964a-c77b-4858-9640-a1995d525472 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T17:55:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-aae39d-predictor namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: 75aebf95-d3c7-43b8-a9c8-9351d4568927 kind: Event lastTimestamp: "2026-07-16T17:56:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:55:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:56:47Z" name: isvc-primary-aae39d-predictor.18c2d6c5dc9af23a namespace: kserve-ci-e2e-test resourceVersion: "20566" uid: 2cfdcf95-93de-4f0a-beb0-46f52c88700b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T17:55:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-aae39d-predictor namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: 75aebf95-d3c7-43b8-a9c8-9351d4568927 kind: Event lastTimestamp: "2026-07-16T17:56:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:55:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:56:47Z" name: isvc-primary-aae39d-predictor.18c2d6c5dd0c6ca7 namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: 9ce382ed-b582-42dd-a792-d8efe20b9527 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-16T17:55:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-aae39d namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: 610cdf46-3334-47f0-b095-473dea74044a kind: Event lastTimestamp: "2026-07-16T17:56:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:56:25Z" name: isvc-primary-aae39d.18c2d6bb5eb61503 namespace: kserve-ci-e2e-test resourceVersion: "20339" uid: 85866589-5de0-48e8-b7ad-d9c21bd7169d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:55:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-aae39d namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: 610cdf46-3334-47f0-b095-473dea74044a kind: Event lastTimestamp: "2026-07-16T17:55:02Z" message: 'Failed to update status for InferenceService "isvc-primary-aae39d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-aae39d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:55:02Z" name: isvc-primary-aae39d.18c2d6bb5f294427 namespace: kserve-ci-e2e-test resourceVersion: "19722" uid: 2292f9e5-1683-45f1-ac35-34ecb471c59e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:55:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-aae39d namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: 610cdf46-3334-47f0-b095-473dea74044a kind: Event lastTimestamp: "2026-07-16T17:55:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-aae39d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:55:02Z" name: isvc-primary-aae39d.18c2d6bb5f29a295 namespace: kserve-ci-e2e-test resourceVersion: "19723" uid: ef2c0135-fab4-49c8-a515-b622e88dbb09 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-aae39d namespace: kserve-ci-e2e-test resourceVersion: "20337" uid: 610cdf46-3334-47f0-b095-473dea74044a kind: Event lastTimestamp: "2026-07-16T17:56:25Z" message: InferenceService [isvc-primary-aae39d] is Ready metadata: creationTimestamp: "2026-07-16T17:56:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:56:25Z" name: isvc-primary-aae39d.18c2d6ce9c88cdc2 namespace: kserve-ci-e2e-test resourceVersion: "20338" uid: 49d104a1-93d1-4fe0-bd6a-a4b1e47384e6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:44:35.964706Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14350" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln to ip-10-0-132-218.ec2.internal metadata: creationTimestamp: "2026-07-16T17:44:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:44:35Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d62976a5cf64 namespace: kserve-ci-e2e-test resourceVersion: "14355" uid: fbd8ffac-5b7c-4972-a219-d22c089b88a7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-699544b6dc-lkd89 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:44:36Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14362" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:44:36Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:44:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:44:36Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d6299021af19 namespace: kserve-ci-e2e-test resourceVersion: "14374" uid: 6fe6bb19-fc9e-44ca-a048-f602fec33326 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:44:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:44:36Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:86207d1cba5aecf8e7b5bca617ed35061e54e289c7778b383f47cc8d53214f1a" metadata: creationTimestamp: "2026-07-16T17:44:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:44:36Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d629919d338f namespace: kserve-ci-e2e-test resourceVersion: "14376" uid: 7ba6ff85-c263-4b7b-96ef-e5d6958a6e6f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:44:39Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:86207d1cba5aecf8e7b5bca617ed35061e54e289c7778b383f47cc8d53214f1a" in 3.276s (3.276s including waiting). Image size: 301882321 bytes.' metadata: creationTimestamp: "2026-07-16T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:44:39Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d62a54e790ab namespace: kserve-ci-e2e-test resourceVersion: "14396" uid: 86952821-05b5-43cb-8cf1-5d033e4e4d20 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:44:39Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-16T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:44:39Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d62a57e05359 namespace: kserve-ci-e2e-test resourceVersion: "14397" uid: 792c26f0-7fc8-4bb4-b9c7-6b4ab2971423 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:44:39Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-16T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:44:39Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d62a58495f1f namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: c1f2c047-fe0b-48b7-823d-a1dec000acac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:44:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:44:44Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1744" metadata: creationTimestamp: "2026-07-16T17:44:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:44:44Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d62b6897d1a9 namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: 3d97c574-0479-4313-b3b3-4b29f9fc5995 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:44:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:44:59Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1744" in 14.706s (14.706s including waiting). Image size: 1558048690 bytes.' metadata: creationTimestamp: "2026-07-16T17:44:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:44:59Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d62ed52c1f47 namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: 6a52942d-7662-44ec-9bfc-5016f4b8307c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:44:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:44:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:44:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:44:59Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d62ed87d5c2d namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 2abc22e7-fb7f-463b-b67e-7ce30cc7e066 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:44:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:44:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:44:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:44:59Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d62ed8f319f1 namespace: kserve-ci-e2e-test resourceVersion: "14544" uid: bffa5b6a-f2b6-4917-9c5c-1ba4164574b7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:44:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:44:59Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-16T17:44:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:44:59Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d62ed90006dc namespace: kserve-ci-e2e-test resourceVersion: "14546" uid: 34b003fa-1412-40e2-a7d2-e3339b7b0b38 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:45:01Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.343s (2.343s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-16T17:45:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:01Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d62f64af7f5a namespace: kserve-ci-e2e-test resourceVersion: "14569" uid: 54f27237-16ba-4bca-830e-8365a129ecaf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:45:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:45:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:01Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d62f68c1d555 namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: f23e5642-cf9d-4a3a-a557-5903bbdea127 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:45:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:45:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:01Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d62f693ab1e7 namespace: kserve-ci-e2e-test resourceVersion: "14572" uid: 7d5ae8a4-0837-463e-95b1-68e63fba6fcb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:45:01Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:42ffb1a3fbaec62e57c71970d909bef17aa5b27605c06189276dc315c300eb9f" metadata: creationTimestamp: "2026-07-16T17:45:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:01Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d62f69481d16 namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: afeb6fde-9fa8-41e0-a9ef-dfed92960588 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:45:04Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:42ffb1a3fbaec62e57c71970d909bef17aa5b27605c06189276dc315c300eb9f" in 2.779s (2.779s including waiting). Image size: 240005750 bytes.' metadata: creationTimestamp: "2026-07-16T17:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:04Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d6300eef12c6 namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: d2e5c41d-779e-40c6-994e-f901eeea9226 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:45:04Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-16T17:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:04Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d63011fd88d5 namespace: kserve-ci-e2e-test resourceVersion: "14591" uid: 9eff2d55-077e-41f9-ad99-c59acbec2049 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:45:04Z" message: Started container agent metadata: creationTimestamp: "2026-07-16T17:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:45:04Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d63012768fea namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: e4407053-516c-4543-a7ec-3be92dd28920 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-16T17:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:46:37Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:37Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d630171ad1b2 namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: 5b899168-6ab9-4b53-8a0e-d4ca6f22a67d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-16T17:45:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:46:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T17:45:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:37Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d63052f3a4bb namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: e1c00c1c-03ae-4be6-a83c-fda1282a95d7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:46:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:31Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d64442ae9577 namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: ceb47c4c-0af0-44fd-8fab-a336726e6bfc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:46:31Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:31Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d64442aef614 namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: 2f862e1c-d41c-438d-ad26-446efa09c8ba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:46:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:31Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d64442af7cb4 namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: 77334541-97d5-4269-8d7b-63353f6da54b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T17:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 27a87488-7dda-494f-a449-6f6cc6f80de9 kind: Event lastTimestamp: "2026-07-16T17:46:46Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:46Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln.18c2d644592a043a namespace: kserve-ci-e2e-test resourceVersion: "15478" uid: 54dfed72-025e-4c7c-a348-83e2a0151335 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:44:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd namespace: kserve-ci-e2e-test resourceVersion: "14338" uid: 9727965b-67e3-4887-97e7-1f9f56c3acfe kind: Event lastTimestamp: "2026-07-16T17:44:35Z" message: 'Created pod: isvc-raw-sklearn-batcher-13951-predictor-9454757dd-v4sln' metadata: creationTimestamp: "2026-07-16T17:44:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:44:35Z" name: isvc-raw-sklearn-batcher-13951-predictor-9454757dd.18c2d6297615cb2e namespace: kserve-ci-e2e-test resourceVersion: "14351" uid: 14004ae3-a3e8-4eda-8b8d-b71d477e85ed reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:44:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-13951-predictor namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 3d28e885-721e-4aba-8629-ee4a434d2c5f kind: Event lastTimestamp: "2026-07-16T17:44:35Z" message: Scaled up replica set isvc-raw-sklearn-batcher-13951-predictor-9454757dd from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:44:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:44:35Z" name: isvc-raw-sklearn-batcher-13951-predictor.18c2d6296ded71e2 namespace: kserve-ci-e2e-test resourceVersion: "14339" uid: 871c7c81-a68f-4fdc-b035-08a52d856fcf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T17:44:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-13951-predictor namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: faa60ccd-2de4-4b49-bf8f-ecb8551a4f24 kind: Event lastTimestamp: "2026-07-16T17:45:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:44:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:45:36Z" name: isvc-raw-sklearn-batcher-13951-predictor.18c2d62cf35fa780 namespace: kserve-ci-e2e-test resourceVersion: "14822" uid: cb179704-87b7-4c03-b4ae-b5a7797a7da1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T17:44:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-13951-predictor namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: faa60ccd-2de4-4b49-bf8f-ecb8551a4f24 kind: Event lastTimestamp: "2026-07-16T17:45:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:44:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:45:36Z" name: isvc-raw-sklearn-batcher-13951-predictor.18c2d62cf3d05da2 namespace: kserve-ci-e2e-test resourceVersion: "14824" uid: 71ab47ca-d798-4f9b-b232-e870fbcacc44 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:45:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-13951-predictor namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: faa60ccd-2de4-4b49-bf8f-ecb8551a4f24 kind: Event lastTimestamp: "2026-07-16T17:46:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:21Z" name: isvc-raw-sklearn-batcher-13951-predictor.18c2d63af13db6d2 namespace: kserve-ci-e2e-test resourceVersion: "15156" uid: d7222fe0-183b-4e84-b655-0306b8b6affb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:45:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-13951-predictor namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: faa60ccd-2de4-4b49-bf8f-ecb8551a4f24 kind: Event lastTimestamp: "2026-07-16T17:46:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:21Z" name: isvc-raw-sklearn-batcher-13951-predictor.18c2d63af1c5be85 namespace: kserve-ci-e2e-test resourceVersion: "15157" uid: d62cb0fe-6970-40be-be76-5f717a7e3cbd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-16T17:44:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-13951 namespace: kserve-ci-e2e-test resourceVersion: "14329" uid: 2dcfa86b-3ba9-41fc-85ab-d639e27ee9d3 kind: Event lastTimestamp: "2026-07-16T17:46:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:44:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:46:17Z" name: isvc-raw-sklearn-batcher-13951.18c2d62973eeaa9e namespace: kserve-ci-e2e-test resourceVersion: "15130" uid: dd29916b-cc72-4b89-9bf0-a6fda6fc3d31 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:44:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-13951 namespace: kserve-ci-e2e-test resourceVersion: "14329" uid: 2dcfa86b-3ba9-41fc-85ab-d639e27ee9d3 kind: Event lastTimestamp: "2026-07-16T17:44:36Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-13951": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-13951": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:44:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:44:36Z" name: isvc-raw-sklearn-batcher-13951.18c2d629744a653a namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: f6474ca4-ddb6-44ad-ae7a-bed7b57df462 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:44:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-13951 namespace: kserve-ci-e2e-test resourceVersion: "14329" uid: 2dcfa86b-3ba9-41fc-85ab-d639e27ee9d3 kind: Event lastTimestamp: "2026-07-16T17:44:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-13951": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:44:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:44:36Z" name: isvc-raw-sklearn-batcher-13951.18c2d629744ab024 namespace: kserve-ci-e2e-test resourceVersion: "14365" uid: b6b2b143-5b67-4207-9a53-3c1097823e16 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-13951 namespace: kserve-ci-e2e-test resourceVersion: "15127" uid: 2dcfa86b-3ba9-41fc-85ab-d639e27ee9d3 kind: Event lastTimestamp: "2026-07-16T17:46:17Z" message: InferenceService [isvc-raw-sklearn-batcher-13951] is Ready metadata: creationTimestamp: "2026-07-16T17:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:46:17Z" name: isvc-raw-sklearn-batcher-13951.18c2d6411e574240 namespace: kserve-ci-e2e-test resourceVersion: "15128" uid: 06b98d4e-4735-4bee-a920-240586669279 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:56:33.043145Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-aae39d-predictor-5d8d6d5c86-2llkd namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: f03f2216-13c3-453c-bad0-0b70c3f64cbe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-aae39d-predictor-5d8d6d5c86-2llkd to ip-10-0-132-218.ec2.internal metadata: creationTimestamp: "2026-07-16T17:56:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:56:33Z" name: isvc-secondary-aae39d-predictor-5d8d6d5c86-2llkd.18c2d6d06bda5c2d namespace: kserve-ci-e2e-test resourceVersion: "20428" uid: c8dfd1e9-1d16-47ee-bb52-6bb0a682d56f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-699544b6dc-lkd89 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:33Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-aae39d-predictor-5d8d6d5c86-2llkd namespace: kserve-ci-e2e-test resourceVersion: "20425" uid: f03f2216-13c3-453c-bad0-0b70c3f64cbe kind: Event lastTimestamp: "2026-07-16T17:56:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-secondary-aae39d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T17:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:33Z" name: isvc-secondary-aae39d-predictor-5d8d6d5c86-2llkd.18c2d6d073214080 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: fd269ab9-4b0d-4c2d-9b97-61cc346fa667 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:34Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-aae39d-predictor-5d8d6d5c86-2llkd namespace: kserve-ci-e2e-test resourceVersion: "20432" uid: f03f2216-13c3-453c-bad0-0b70c3f64cbe kind: Event lastTimestamp: "2026-07-16T17:56:34Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:56:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:56:34Z" name: isvc-secondary-aae39d-predictor-5d8d6d5c86-2llkd.18c2d6d0a9f1a1de namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: d44ebde3-5f77-444a-b11e-30bb37aa05d4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:56:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-aae39d-predictor-5d8d6d5c86-2llkd namespace: kserve-ci-e2e-test resourceVersion: "20425" uid: f03f2216-13c3-453c-bad0-0b70c3f64cbe kind: Event lastTimestamp: "2026-07-16T17:56:40Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:86207d1cba5aecf8e7b5bca617ed35061e54e289c7778b383f47cc8d53214f1a" already present on machine metadata: creationTimestamp: "2026-07-16T17:56:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:40Z" name: isvc-secondary-aae39d-predictor-5d8d6d5c86-2llkd.18c2d6d0abad995f namespace: kserve-ci-e2e-test resourceVersion: "20504" uid: e9d9105b-df0a-42eb-a47a-7bf84c516086 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:56:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-aae39d-predictor-5d8d6d5c86-2llkd namespace: kserve-ci-e2e-test resourceVersion: "20425" uid: f03f2216-13c3-453c-bad0-0b70c3f64cbe kind: Event lastTimestamp: "2026-07-16T17:56:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-16T17:56:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:40Z" name: isvc-secondary-aae39d-predictor-5d8d6d5c86-2llkd.18c2d6d0aed4a0f1 namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 0e80c44d-bb5b-4a06-a137-44fc93e45de0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:56:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-aae39d-predictor-5d8d6d5c86-2llkd namespace: kserve-ci-e2e-test resourceVersion: "20425" uid: f03f2216-13c3-453c-bad0-0b70c3f64cbe kind: Event lastTimestamp: "2026-07-16T17:56:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-16T17:56:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:40Z" name: isvc-secondary-aae39d-predictor-5d8d6d5c86-2llkd.18c2d6d0af504d4d namespace: kserve-ci-e2e-test resourceVersion: "20507" uid: e930ba86-eead-4a34-89b4-dde53d8b3c72 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-aae39d-predictor-5d8d6d5c86-2llkd namespace: kserve-ci-e2e-test resourceVersion: "20425" uid: f03f2216-13c3-453c-bad0-0b70c3f64cbe kind: Event lastTimestamp: "2026-07-16T17:56:46Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-aae39d-predictor-5d8d6d5c86-2llkd_kserve-ci-e2e-test(f03f2216-13c3-453c-bad0-0b70c3f64cbe) metadata: creationTimestamp: "2026-07-16T17:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:56:46Z" name: isvc-secondary-aae39d-predictor-5d8d6d5c86-2llkd.18c2d6d390df554c namespace: kserve-ci-e2e-test resourceVersion: "20552" uid: acb1c51a-bb70-4538-9334-52e3ba365b40 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-aae39d-predictor-5d8d6d5c86 namespace: kserve-ci-e2e-test resourceVersion: "20413" uid: 0ea627f1-46ed-4b4e-a4c3-b8a93ad65dca kind: Event lastTimestamp: "2026-07-16T17:56:33Z" message: 'Created pod: isvc-secondary-aae39d-predictor-5d8d6d5c86-2llkd' metadata: creationTimestamp: "2026-07-16T17:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:56:33Z" name: isvc-secondary-aae39d-predictor-5d8d6d5c86.18c2d6d06b6820ad namespace: kserve-ci-e2e-test resourceVersion: "20426" uid: 32d33fe8-c27b-462f-8f03-efdd3ff98063 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-aae39d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20411" uid: 8aba38ce-9edb-4923-9b5d-21f0be769a25 kind: Event lastTimestamp: "2026-07-16T17:56:32Z" message: Scaled up replica set isvc-secondary-aae39d-predictor-5d8d6d5c86 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:56:32Z" name: isvc-secondary-aae39d-predictor.18c2d6d066407ace namespace: kserve-ci-e2e-test resourceVersion: "20414" uid: 8d626fc5-30d4-4e0a-882b-24733f195fac reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-aae39d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20412" uid: bd9f7461-cd94-4df3-9b34-ef43645031fc kind: Event lastTimestamp: "2026-07-16T17:56:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:56:47Z" name: isvc-secondary-aae39d-predictor.18c2d6d3e489d914 namespace: kserve-ci-e2e-test resourceVersion: "20568" uid: 61ac8e9b-94f0-4383-9578-f2d351c47e11 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:56:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-aae39d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20412" uid: bd9f7461-cd94-4df3-9b34-ef43645031fc kind: Event lastTimestamp: "2026-07-16T17:56:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:56:47Z" name: isvc-secondary-aae39d-predictor.18c2d6d3e50f4c72 namespace: kserve-ci-e2e-test resourceVersion: "20570" uid: 744b806a-85f4-41df-9a68-d3b8b7aae7ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-07-16T17:56:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-aae39d namespace: kserve-ci-e2e-test resourceVersion: "20404" uid: 87f3486a-3665-4eff-b89f-60d3bb6d48f3 kind: Event lastTimestamp: "2026-07-16T17:56:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:56:46Z" name: isvc-secondary-aae39d.18c2d6d06645a5ed namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: 435abe58-d818-45c5-9e7a-e95b7eb8ef6c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:56:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-aae39d namespace: kserve-ci-e2e-test resourceVersion: "20404" uid: 87f3486a-3665-4eff-b89f-60d3bb6d48f3 kind: Event lastTimestamp: "2026-07-16T17:56:33Z" message: 'Failed to update status for InferenceService "isvc-secondary-aae39d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-aae39d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:56:33Z" name: isvc-secondary-aae39d.18c2d6d0672d534a namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: efc6c9cf-7d0b-4cfa-9d66-ad66f9ce88d6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:56:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-aae39d namespace: kserve-ci-e2e-test resourceVersion: "20404" uid: 87f3486a-3665-4eff-b89f-60d3bb6d48f3 kind: Event lastTimestamp: "2026-07-16T17:56:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-aae39d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:56:33Z" name: isvc-secondary-aae39d.18c2d6d0672daa68 namespace: kserve-ci-e2e-test resourceVersion: "20435" uid: 381f4a75-bc09-4ff1-b264-72798ec87455 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:46:31.161934Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: b91b1aca-21ae-4cf9-83ae-d672d41d7d9b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v to ip-10-0-132-218.ec2.internal metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:46:31Z" name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v.18c2d64448efe48e namespace: kserve-ci-e2e-test resourceVersion: "15276" uid: a9d37ce3-71d9-404b-8dc8-bdfc0090c9ce reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-699544b6dc-lkd89 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v namespace: kserve-ci-e2e-test resourceVersion: "15284" uid: b91b1aca-21ae-4cf9-83ae-d672d41d7d9b kind: Event lastTimestamp: "2026-07-16T17:46:31Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:46:31Z" name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v.18c2d6446336c0a9 namespace: kserve-ci-e2e-test resourceVersion: "15316" uid: c185a021-120e-4202-86ae-ea30bf633d26 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: b91b1aca-21ae-4cf9-83ae-d672d41d7d9b kind: Event lastTimestamp: "2026-07-16T17:46:31Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:86207d1cba5aecf8e7b5bca617ed35061e54e289c7778b383f47cc8d53214f1a" already present on machine metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:31Z" name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v.18c2d64464a8290d namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: 0271777d-1dc2-4441-9014-707f9aae8fed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: b91b1aca-21ae-4cf9-83ae-d672d41d7d9b kind: Event lastTimestamp: "2026-07-16T17:46:31Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:31Z" name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v.18c2d64467f03fe8 namespace: kserve-ci-e2e-test resourceVersion: "15320" uid: 105ad609-035e-4ac5-867d-e6a6dba2da12 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: b91b1aca-21ae-4cf9-83ae-d672d41d7d9b kind: Event lastTimestamp: "2026-07-16T17:46:31Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:31Z" name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v.18c2d6446857306b namespace: kserve-ci-e2e-test resourceVersion: "15321" uid: ac938865-b5cf-4ccc-bee5-ad79b147d352 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: b91b1aca-21ae-4cf9-83ae-d672d41d7d9b kind: Event lastTimestamp: "2026-07-16T17:46:35Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T17:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:35Z" name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v.18c2d6455eb402c7 namespace: kserve-ci-e2e-test resourceVersion: "15372" uid: d376a0f6-a75c-4018-b84b-e61e1e15409a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: b91b1aca-21ae-4cf9-83ae-d672d41d7d9b kind: Event lastTimestamp: "2026-07-16T17:46:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:35Z" name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v.18c2d64561a49f7f namespace: kserve-ci-e2e-test resourceVersion: "15375" uid: 32ce6524-4004-4d78-86b8-f218e60d956d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: b91b1aca-21ae-4cf9-83ae-d672d41d7d9b kind: Event lastTimestamp: "2026-07-16T17:46:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:35Z" name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v.18c2d645620a9dda namespace: kserve-ci-e2e-test resourceVersion: "15376" uid: f7b50ebc-c501-4fc2-9ee6-c739e2906148 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: b91b1aca-21ae-4cf9-83ae-d672d41d7d9b kind: Event lastTimestamp: "2026-07-16T17:46:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:35Z" name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v.18c2d6456215c06b namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: a7dc35ae-fb92-41f4-8db6-79a1d3fcbaec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: b91b1aca-21ae-4cf9-83ae-d672d41d7d9b kind: Event lastTimestamp: "2026-07-16T17:46:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:35Z" name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v.18c2d6456707c050 namespace: kserve-ci-e2e-test resourceVersion: "15382" uid: 672d2e42-91e7-4656-8b28-218c1e10555c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: b91b1aca-21ae-4cf9-83ae-d672d41d7d9b kind: Event lastTimestamp: "2026-07-16T17:46:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:35Z" name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v.18c2d64567771151 namespace: kserve-ci-e2e-test resourceVersion: "15383" uid: 6c2f5464-db28-4566-8a1c-a3a34a777419 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-16T17:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: b91b1aca-21ae-4cf9-83ae-d672d41d7d9b kind: Event lastTimestamp: "2026-07-16T17:48:23Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:23Z" name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v.18c2d645d6ee62b8 namespace: kserve-ci-e2e-test resourceVersion: "16395" uid: 3124ae64-cd01-4ed0-8149-b50e30124c13 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: b91b1aca-21ae-4cf9-83ae-d672d41d7d9b kind: Event lastTimestamp: "2026-07-16T17:48:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:21Z" name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v.18c2d65df44ab678 namespace: kserve-ci-e2e-test resourceVersion: "16281" uid: e7aead09-abc1-4290-b207-bbd7c49147c3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: b91b1aca-21ae-4cf9-83ae-d672d41d7d9b kind: Event lastTimestamp: "2026-07-16T17:48:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:21Z" name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v.18c2d65df44b4a8b namespace: kserve-ci-e2e-test resourceVersion: "16282" uid: 4e860330-dd5e-469b-ac6d-c3435a7a499a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: b91b1aca-21ae-4cf9-83ae-d672d41d7d9b kind: Event lastTimestamp: "2026-07-16T17:48:23Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:48:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:23Z" name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v.18c2d65e85598d95 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: 8d82e741-6006-4278-a928-acc8de939e38 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768 namespace: kserve-ci-e2e-test resourceVersion: "15252" uid: 7a71ae1b-7d21-47c8-8522-248d6959501f kind: Event lastTimestamp: "2026-07-16T17:46:31Z" message: 'Created pod: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768-4dv6v' metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:31Z" name: isvc-sklearn-graph-raw-1404c-predictor-dbdf54768.18c2d644488137d0 namespace: kserve-ci-e2e-test resourceVersion: "15271" uid: e48c47d0-2cd2-4691-8aa7-805b00ad2486 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-1404c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: df693e0d-5747-4788-9a7a-c64a22d72e56 kind: Event lastTimestamp: "2026-07-16T17:46:31Z" message: Scaled up replica set isvc-sklearn-graph-raw-1404c-predictor-dbdf54768 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:31Z" name: isvc-sklearn-graph-raw-1404c-predictor.18c2d644437482c1 namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: 3b38e73e-dbb2-4741-9485-62ad79ac4cd0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:46:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-1404c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15250" uid: 57a838ce-86d8-4232-afbe-bd655308b3f1 kind: Event lastTimestamp: "2026-07-16T17:47:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:01Z" name: isvc-sklearn-graph-raw-1404c-predictor.18c2d647c1d6397b namespace: kserve-ci-e2e-test resourceVersion: "15588" uid: 3974b6e4-53e8-403d-88f4-8e67c5927193 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:46:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-1404c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15250" uid: 57a838ce-86d8-4232-afbe-bd655308b3f1 kind: Event lastTimestamp: "2026-07-16T17:47:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:01Z" name: isvc-sklearn-graph-raw-1404c-predictor.18c2d647c23f83ae namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: 38e1509a-2845-442e-9ec9-828b507074fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T17:47:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-1404c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15469" uid: 57a838ce-86d8-4232-afbe-bd655308b3f1 kind: Event lastTimestamp: "2026-07-16T17:48:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:16Z" name: isvc-sklearn-graph-raw-1404c-predictor.18c2d64ec0d4d80d namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: cfbd7ad7-71f5-4c9a-9d58-5cddf731f718 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T17:47:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-1404c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15469" uid: 57a838ce-86d8-4232-afbe-bd655308b3f1 kind: Event lastTimestamp: "2026-07-16T17:48:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:16Z" name: isvc-sklearn-graph-raw-1404c-predictor.18c2d64ec14c81cc namespace: kserve-ci-e2e-test resourceVersion: "16191" uid: 7c4c9a8b-0ae5-44d3-aabc-613ea3a1f67c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-16T17:46:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-1404c namespace: kserve-ci-e2e-test resourceVersion: "15226" uid: 210fa9c3-5a88-4d79-a9d8-bf6f525baac4 kind: Event lastTimestamp: "2026-07-16T17:47:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:47:53Z" name: isvc-sklearn-graph-raw-1404c.18c2d6444378ed4a namespace: kserve-ci-e2e-test resourceVersion: "15988" uid: 0845e766-1bce-4845-ad83-abb6c37a5c58 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-1404c namespace: kserve-ci-e2e-test resourceVersion: "15226" uid: 210fa9c3-5a88-4d79-a9d8-bf6f525baac4 kind: Event lastTimestamp: "2026-07-16T17:46:31Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-1404c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-1404c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:46:31Z" name: isvc-sklearn-graph-raw-1404c.18c2d644441ce92d namespace: kserve-ci-e2e-test resourceVersion: "15259" uid: 9a3df3d8-c33f-427b-97b8-351d9e474400 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:47:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-1404c namespace: kserve-ci-e2e-test resourceVersion: "15985" uid: 210fa9c3-5a88-4d79-a9d8-bf6f525baac4 kind: Event lastTimestamp: "2026-07-16T17:47:53Z" message: InferenceService [isvc-sklearn-graph-raw-1404c] is Ready metadata: creationTimestamp: "2026-07-16T17:47:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:47:53Z" name: isvc-sklearn-graph-raw-1404c.18c2d6578f4de266 namespace: kserve-ci-e2e-test resourceVersion: "15986" uid: 777f1bcd-d734-4c50-b499-7405ebc2e0e8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:48:21.521701Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: 580d18b7-efd9-410c-acda-0e237b5941d8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb to ip-10-0-132-218.ec2.internal metadata: creationTimestamp: "2026-07-16T17:48:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:48:21Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb.18c2d65dfae45029 namespace: kserve-ci-e2e-test resourceVersion: "16295" uid: dfa15a0e-b47a-4ce3-a1c4-6c99a3ded3e1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-699544b6dc-lkd89 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:21Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb namespace: kserve-ci-e2e-test resourceVersion: "16306" uid: 580d18b7-efd9-410c-acda-0e237b5941d8 kind: Event lastTimestamp: "2026-07-16T17:48:21Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:48:21Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb.18c2d65e1528c638 namespace: kserve-ci-e2e-test resourceVersion: "16358" uid: 2cbd35ba-a62d-4c17-b21e-fb6e79fafa5a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb namespace: kserve-ci-e2e-test resourceVersion: "16291" uid: 580d18b7-efd9-410c-acda-0e237b5941d8 kind: Event lastTimestamp: "2026-07-16T17:48:21Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:86207d1cba5aecf8e7b5bca617ed35061e54e289c7778b383f47cc8d53214f1a" already present on machine metadata: creationTimestamp: "2026-07-16T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:21Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb.18c2d65e16a996fd namespace: kserve-ci-e2e-test resourceVersion: "16361" uid: 90d3fad6-e080-4cf1-a1ec-e2ea0245ca59 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb namespace: kserve-ci-e2e-test resourceVersion: "16291" uid: 580d18b7-efd9-410c-acda-0e237b5941d8 kind: Event lastTimestamp: "2026-07-16T17:48:22Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-16T17:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:22Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb.18c2d65e196ea8aa namespace: kserve-ci-e2e-test resourceVersion: "16363" uid: 91ff0949-27f0-4b7a-a6ea-598e612c695c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb namespace: kserve-ci-e2e-test resourceVersion: "16291" uid: 580d18b7-efd9-410c-acda-0e237b5941d8 kind: Event lastTimestamp: "2026-07-16T17:48:22Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-16T17:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:22Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb.18c2d65e19d86daf namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 2a147e6f-92d2-45a2-a567-f3f28f7eb5ce reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb namespace: kserve-ci-e2e-test resourceVersion: "16291" uid: 580d18b7-efd9-410c-acda-0e237b5941d8 kind: Event lastTimestamp: "2026-07-16T17:48:26Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T17:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:26Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb.18c2d65f1baaedaa namespace: kserve-ci-e2e-test resourceVersion: "16421" uid: 977ed79a-e26d-4985-a40f-a6ef33a18d6c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb namespace: kserve-ci-e2e-test resourceVersion: "16291" uid: 580d18b7-efd9-410c-acda-0e237b5941d8 kind: Event lastTimestamp: "2026-07-16T17:48:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:26Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb.18c2d65f1f7d21f1 namespace: kserve-ci-e2e-test resourceVersion: "16426" uid: d4c7456a-3cd4-4abb-80ed-b1ef10473f46 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb namespace: kserve-ci-e2e-test resourceVersion: "16291" uid: 580d18b7-efd9-410c-acda-0e237b5941d8 kind: Event lastTimestamp: "2026-07-16T17:48:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:26Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb.18c2d65f1fef67e3 namespace: kserve-ci-e2e-test resourceVersion: "16428" uid: 55c17119-ade0-4188-88c7-68455ca3629e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb namespace: kserve-ci-e2e-test resourceVersion: "16291" uid: 580d18b7-efd9-410c-acda-0e237b5941d8 kind: Event lastTimestamp: "2026-07-16T17:48:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:26Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb.18c2d65f1ffb46ff namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: d2d93430-7c0b-4d01-a830-34414e6f10a0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb namespace: kserve-ci-e2e-test resourceVersion: "16291" uid: 580d18b7-efd9-410c-acda-0e237b5941d8 kind: Event lastTimestamp: "2026-07-16T17:48:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:26Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb.18c2d65f23590e70 namespace: kserve-ci-e2e-test resourceVersion: "16432" uid: a27166cc-71cc-465a-ac9c-acd74f32ad51 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb namespace: kserve-ci-e2e-test resourceVersion: "16291" uid: 580d18b7-efd9-410c-acda-0e237b5941d8 kind: Event lastTimestamp: "2026-07-16T17:48:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:26Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb.18c2d65f23c94aa8 namespace: kserve-ci-e2e-test resourceVersion: "16433" uid: 4782718b-5f47-4f0b-a58d-a0552deea7b5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-16T17:48:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb namespace: kserve-ci-e2e-test resourceVersion: "16291" uid: 580d18b7-efd9-410c-acda-0e237b5941d8 kind: Event lastTimestamp: "2026-07-16T17:50:14Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:14Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb.18c2d65f94151aaf namespace: kserve-ci-e2e-test resourceVersion: "17402" uid: a9c26633-2aa5-4c11-8b64-79cb436136a8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb namespace: kserve-ci-e2e-test resourceVersion: "16291" uid: 580d18b7-efd9-410c-acda-0e237b5941d8 kind: Event lastTimestamp: "2026-07-16T17:50:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:11Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb.18c2d677a8b08d00 namespace: kserve-ci-e2e-test resourceVersion: "17340" uid: 3e38d578-5414-42b5-9194-4fdcf6fec73a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb namespace: kserve-ci-e2e-test resourceVersion: "16291" uid: 580d18b7-efd9-410c-acda-0e237b5941d8 kind: Event lastTimestamp: "2026-07-16T17:50:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:11Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb.18c2d677a8b184b3 namespace: kserve-ci-e2e-test resourceVersion: "17341" uid: 6695b20b-ca01-44ef-b232-7f6a77419be2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb namespace: kserve-ci-e2e-test resourceVersion: "16291" uid: 580d18b7-efd9-410c-acda-0e237b5941d8 kind: Event lastTimestamp: "2026-07-16T17:50:14Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp 10.133.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:14Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb.18c2d678426288e5 namespace: kserve-ci-e2e-test resourceVersion: "17401" uid: 71fb3724-953e-4448-86b2-b5f02be5b315 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4 namespace: kserve-ci-e2e-test resourceVersion: "16275" uid: cfe13635-8ac7-4660-9e0a-dd2cb8341760 kind: Event lastTimestamp: "2026-07-16T17:48:21Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4-bxwtb' metadata: creationTimestamp: "2026-07-16T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:21Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4.18c2d65dfa77907b namespace: kserve-ci-e2e-test resourceVersion: "16292" uid: e4775dc7-1a1b-46cb-98da-aacf29d430d8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-57df2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 70d53b36-2f95-47e3-99b1-93a39a4124ac kind: Event lastTimestamp: "2026-07-16T17:48:21Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-57df2-predictor-6bc9ffd7d4 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:21Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor.18c2d65df3daefa9 namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: a0fc9113-e77d-49a6-83c1-a40ea807b4d0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:48:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-57df2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16274" uid: 5f09697e-af05-4d92-ae61-676419882866 kind: Event lastTimestamp: "2026-07-16T17:49:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:49:06Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor.18c2d66173ba625c namespace: kserve-ci-e2e-test resourceVersion: "16747" uid: 718da2df-d06b-462d-97c0-cf017ea80716 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:48:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-57df2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16274" uid: 5f09697e-af05-4d92-ae61-676419882866 kind: Event lastTimestamp: "2026-07-16T17:49:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:49:06Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor.18c2d661745818b4 namespace: kserve-ci-e2e-test resourceVersion: "16749" uid: c06aa461-58d8-4b7e-a75c-a215d9bc0ecf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T17:48:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-57df2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16516" uid: 5f09697e-af05-4d92-ae61-676419882866 kind: Event lastTimestamp: "2026-07-16T17:50:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:48:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:50:06Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor.18c2d664f3391cdc namespace: kserve-ci-e2e-test resourceVersion: "17243" uid: 6d9b230a-59b5-4e3a-8f63-6cb9ad3a7be6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T17:48:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-57df2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16516" uid: 5f09697e-af05-4d92-ae61-676419882866 kind: Event lastTimestamp: "2026-07-16T17:50:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:48:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:50:06Z" name: isvc-sklearn-graph-raw-hpa-57df2-predictor.18c2d664f3b04b02 namespace: kserve-ci-e2e-test resourceVersion: "17244" uid: d2857ad0-c24c-4c45-8812-7353188c7ab3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-16T17:48:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-57df2 namespace: kserve-ci-e2e-test resourceVersion: "16252" uid: 72d5ca4e-3358-4631-8323-2932e7edd998 kind: Event lastTimestamp: "2026-07-16T17:49:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:49:44Z" name: isvc-sklearn-graph-raw-hpa-57df2.18c2d65df3dbd76e namespace: kserve-ci-e2e-test resourceVersion: "17044" uid: 44495b10-5888-4247-8d8f-f8e55fb51bd1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-57df2 namespace: kserve-ci-e2e-test resourceVersion: "16252" uid: 72d5ca4e-3358-4631-8323-2932e7edd998 kind: Event lastTimestamp: "2026-07-16T17:48:21Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-57df2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-57df2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:48:21Z" name: isvc-sklearn-graph-raw-hpa-57df2.18c2d65df4d2a7b3 namespace: kserve-ci-e2e-test resourceVersion: "16283" uid: 30705132-8dd6-4b9d-a91a-594e6759b900 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:49:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-57df2 namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: 72d5ca4e-3358-4631-8323-2932e7edd998 kind: Event lastTimestamp: "2026-07-16T17:49:44Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-57df2] is Ready metadata: creationTimestamp: "2026-07-16T17:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:49:44Z" name: isvc-sklearn-graph-raw-hpa-57df2.18c2d6714f5a7234 namespace: kserve-ci-e2e-test resourceVersion: "17042" uid: cf512b19-7982-4b59-ade2-7f6d63f59a42 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:52:07.286656Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6 namespace: kserve-ci-e2e-test resourceVersion: "18364" uid: 1049ca5f-1d2d-4f8e-adc8-1ecf32019676 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6 to ip-10-0-132-218.ec2.internal metadata: creationTimestamp: "2026-07-16T17:52:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:52:07Z" name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6.18c2d6928b882278 namespace: kserve-ci-e2e-test resourceVersion: "18373" uid: 7ac3fa3f-1b5a-4843-b07b-22451fed3b53 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-699544b6dc-lkd89 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:52:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6 namespace: kserve-ci-e2e-test resourceVersion: "18385" uid: 1049ca5f-1d2d-4f8e-adc8-1ecf32019676 kind: Event lastTimestamp: "2026-07-16T17:52:07Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:52:07Z" name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6.18c2d692a597897c namespace: kserve-ci-e2e-test resourceVersion: "18395" uid: 83642d16-2532-4942-bda6-977f2ea812e3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6 namespace: kserve-ci-e2e-test resourceVersion: "18371" uid: 1049ca5f-1d2d-4f8e-adc8-1ecf32019676 kind: Event lastTimestamp: "2026-07-16T17:52:07Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:86207d1cba5aecf8e7b5bca617ed35061e54e289c7778b383f47cc8d53214f1a" already present on machine metadata: creationTimestamp: "2026-07-16T17:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:52:07Z" name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6.18c2d692a74ff37e namespace: kserve-ci-e2e-test resourceVersion: "18397" uid: e81d9af6-32b8-47a2-af8f-18998a847587 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6 namespace: kserve-ci-e2e-test resourceVersion: "18371" uid: 1049ca5f-1d2d-4f8e-adc8-1ecf32019676 kind: Event lastTimestamp: "2026-07-16T17:52:07Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-16T17:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:52:07Z" name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6.18c2d692aa980067 namespace: kserve-ci-e2e-test resourceVersion: "18399" uid: 16a421a8-9cbe-4ea0-9569-a26a49c135c1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6 namespace: kserve-ci-e2e-test resourceVersion: "18371" uid: 1049ca5f-1d2d-4f8e-adc8-1ecf32019676 kind: Event lastTimestamp: "2026-07-16T17:52:07Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-16T17:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:52:07Z" name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6.18c2d692ab11cbf1 namespace: kserve-ci-e2e-test resourceVersion: "18400" uid: 1effd4a0-637d-402a-9162-77e6894ff3cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:52:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6 namespace: kserve-ci-e2e-test resourceVersion: "18371" uid: 1049ca5f-1d2d-4f8e-adc8-1ecf32019676 kind: Event lastTimestamp: "2026-07-16T17:52:12Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T17:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:52:12Z" name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6.18c2d693baf77f63 namespace: kserve-ci-e2e-test resourceVersion: "18445" uid: a40018c5-6bf2-4679-b09e-88642fa7a1df reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:52:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6 namespace: kserve-ci-e2e-test resourceVersion: "18371" uid: 1049ca5f-1d2d-4f8e-adc8-1ecf32019676 kind: Event lastTimestamp: "2026-07-16T17:52:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:52:12Z" name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6.18c2d693bdfa6104 namespace: kserve-ci-e2e-test resourceVersion: "18447" uid: 7c6914c1-454a-4eb8-8869-b07a89ad377e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:52:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6 namespace: kserve-ci-e2e-test resourceVersion: "18371" uid: 1049ca5f-1d2d-4f8e-adc8-1ecf32019676 kind: Event lastTimestamp: "2026-07-16T17:52:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:52:12Z" name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6.18c2d693be725103 namespace: kserve-ci-e2e-test resourceVersion: "18448" uid: 499928a6-8701-4bd7-ae70-b8bfd3ae553c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:52:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6 namespace: kserve-ci-e2e-test resourceVersion: "18371" uid: 1049ca5f-1d2d-4f8e-adc8-1ecf32019676 kind: Event lastTimestamp: "2026-07-16T17:52:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:52:12Z" name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6.18c2d693be7e950f namespace: kserve-ci-e2e-test resourceVersion: "18449" uid: 6e5e9d44-47c1-4278-8a9c-76b3763d0631 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:52:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6 namespace: kserve-ci-e2e-test resourceVersion: "18371" uid: 1049ca5f-1d2d-4f8e-adc8-1ecf32019676 kind: Event lastTimestamp: "2026-07-16T17:52:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:52:12Z" name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6.18c2d693c1b6d218 namespace: kserve-ci-e2e-test resourceVersion: "18452" uid: 4002e168-8002-4a26-8f23-ea5a1867af00 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:52:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6 namespace: kserve-ci-e2e-test resourceVersion: "18371" uid: 1049ca5f-1d2d-4f8e-adc8-1ecf32019676 kind: Event lastTimestamp: "2026-07-16T17:52:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:52:12Z" name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6.18c2d693c22fe0b0 namespace: kserve-ci-e2e-test resourceVersion: "18453" uid: 03f51b82-7706-4186-8150-b546cf04f144 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 19 eventTime: null firstTimestamp: "2026-07-16T17:52:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6 namespace: kserve-ci-e2e-test resourceVersion: "18371" uid: 1049ca5f-1d2d-4f8e-adc8-1ecf32019676 kind: Event lastTimestamp: "2026-07-16T17:55:04Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:52:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:55:04Z" name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6.18c2d693f70c0f45 namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 4b1a822f-7675-4cb2-b770-37fef205aa5c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:54:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6 namespace: kserve-ci-e2e-test resourceVersion: "18371" uid: 1049ca5f-1d2d-4f8e-adc8-1ecf32019676 kind: Event lastTimestamp: "2026-07-16T17:54:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:54:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:54:57Z" name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6.18c2d6ba29f53bd3 namespace: kserve-ci-e2e-test resourceVersion: "19649" uid: 4827231d-0b39-4c88-936a-e93bfc4d21b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:54:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6 namespace: kserve-ci-e2e-test resourceVersion: "18371" uid: 1049ca5f-1d2d-4f8e-adc8-1ecf32019676 kind: Event lastTimestamp: "2026-07-16T17:54:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:54:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:54:57Z" name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6.18c2d6ba29f5a70f namespace: kserve-ci-e2e-test resourceVersion: "19650" uid: 30d8d19a-8fcc-400a-a0ed-98a28c66db60 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6 namespace: kserve-ci-e2e-test resourceVersion: "18371" uid: 1049ca5f-1d2d-4f8e-adc8-1ecf32019676 kind: Event lastTimestamp: "2026-07-16T17:55:04Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp 10.133.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:55:04Z" name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6.18c2d6ba9d9ff7d5 namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: 3af16c71-2652-416e-8ca4-fafc25b9ce58 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:52:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db namespace: kserve-ci-e2e-test resourceVersion: "18344" uid: 0e007290-9640-439e-a58b-4743c18d0423 kind: Event lastTimestamp: "2026-07-16T17:52:07Z" message: 'Created pod: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db-fp5w6' metadata: creationTimestamp: "2026-07-16T17:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:52:07Z" name: isvc-sklearn-scale-raw-b0db6-predictor-667dc794db.18c2d6928af95794 namespace: kserve-ci-e2e-test resourceVersion: "18367" uid: ca3eabde-af1d-4a1c-8aee-8645df75bc18 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:52:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-b0db6-predictor namespace: kserve-ci-e2e-test resourceVersion: "18340" uid: dbd262d4-e28b-4c6c-b742-fdc389819c57 kind: Event lastTimestamp: "2026-07-16T17:52:07Z" message: Scaled up replica set isvc-sklearn-scale-raw-b0db6-predictor-667dc794db from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:52:07Z" name: isvc-sklearn-scale-raw-b0db6-predictor.18c2d69285a7113d namespace: kserve-ci-e2e-test resourceVersion: "18347" uid: 8c21d3c1-9999-4a31-bc1e-15a3f6838873 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:52:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-b0db6-predictor namespace: kserve-ci-e2e-test resourceVersion: "18343" uid: f98eced7-e492-4d6f-b4d2-ad4011535e07 kind: Event lastTimestamp: "2026-07-16T17:52:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:52:52Z" name: isvc-sklearn-scale-raw-b0db6-predictor.18c2d696060202b2 namespace: kserve-ci-e2e-test resourceVersion: "18754" uid: 4dd49158-05df-4037-9cb9-a48eef787d69 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:52:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-b0db6-predictor namespace: kserve-ci-e2e-test resourceVersion: "18343" uid: f98eced7-e492-4d6f-b4d2-ad4011535e07 kind: Event lastTimestamp: "2026-07-16T17:52:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:52:52Z" name: isvc-sklearn-scale-raw-b0db6-predictor.18c2d69606dcdc1b namespace: kserve-ci-e2e-test resourceVersion: "18755" uid: d909f2c5-2d38-4ac8-8340-965a67b20cf7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-16T17:52:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-b0db6-predictor namespace: kserve-ci-e2e-test resourceVersion: "18535" uid: f98eced7-e492-4d6f-b4d2-ad4011535e07 kind: Event lastTimestamp: "2026-07-16T17:54:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:52:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:54:52Z" name: isvc-sklearn-scale-raw-b0db6-predictor.18c2d69985cf87d1 namespace: kserve-ci-e2e-test resourceVersion: "19583" uid: 434b279e-eee8-40df-a3d3-8d8ae0f03390 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-16T17:52:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-b0db6-predictor namespace: kserve-ci-e2e-test resourceVersion: "18535" uid: f98eced7-e492-4d6f-b4d2-ad4011535e07 kind: Event lastTimestamp: "2026-07-16T17:54:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:52:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:54:52Z" name: isvc-sklearn-scale-raw-b0db6-predictor.18c2d69986624404 namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: cb9428d0-eb1b-46d6-b794-26622f18ce2e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-16T17:52:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-b0db6 namespace: kserve-ci-e2e-test resourceVersion: "18322" uid: c9a3f061-5dab-489d-af84-02f4231088df kind: Event lastTimestamp: "2026-07-16T17:54:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:54:54Z" name: isvc-sklearn-scale-raw-b0db6.18c2d69285b06ab3 namespace: kserve-ci-e2e-test resourceVersion: "19610" uid: 81b61ba9-e3aa-4f5d-9a7b-2ffefe87101c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:52:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-b0db6 namespace: kserve-ci-e2e-test resourceVersion: "18322" uid: c9a3f061-5dab-489d-af84-02f4231088df kind: Event lastTimestamp: "2026-07-16T17:52:07Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-b0db6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-b0db6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:52:07Z" name: isvc-sklearn-scale-raw-b0db6.18c2d69286557ea8 namespace: kserve-ci-e2e-test resourceVersion: "18352" uid: 0c9385d7-56a0-455f-91ca-fcba8ed7e33a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:54:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-b0db6 namespace: kserve-ci-e2e-test resourceVersion: "19607" uid: c9a3f061-5dab-489d-af84-02f4231088df kind: Event lastTimestamp: "2026-07-16T17:54:54Z" message: InferenceService [isvc-sklearn-scale-raw-b0db6] is Ready metadata: creationTimestamp: "2026-07-16T17:54:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:54:54Z" name: isvc-sklearn-scale-raw-b0db6.18c2d6b98b8abb9d namespace: kserve-ci-e2e-test resourceVersion: "19608" uid: 0f3517da-ee7e-4e94-a652-b3de012815d2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:46:31.250456Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 186e8cfd-e44b-4914-8c08-6c598764659b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb to ip-10-0-132-218.ec2.internal metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:46:31Z" name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb.18c2d6444e36a245 namespace: kserve-ci-e2e-test resourceVersion: "15301" uid: 99f52aea-f00f-41de-9580-e1a66793ab38 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-699544b6dc-lkd89 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 186e8cfd-e44b-4914-8c08-6c598764659b kind: Event lastTimestamp: "2026-07-16T17:46:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-1404c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:31Z" name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb.18c2d6445aef4a0e namespace: kserve-ci-e2e-test resourceVersion: "15315" uid: 9dc9a41a-57b9-4755-9d41-470ab18bbebb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:32Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb namespace: kserve-ci-e2e-test resourceVersion: "15305" uid: 186e8cfd-e44b-4914-8c08-6c598764659b kind: Event lastTimestamp: "2026-07-16T17:46:32Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:46:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:46:32Z" name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb.18c2d6448bd20e70 namespace: kserve-ci-e2e-test resourceVersion: "15337" uid: d548efba-4ad2-40cf-9fbc-a7d924f9bf33 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 186e8cfd-e44b-4914-8c08-6c598764659b kind: Event lastTimestamp: "2026-07-16T17:46:32Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:86207d1cba5aecf8e7b5bca617ed35061e54e289c7778b383f47cc8d53214f1a" already present on machine metadata: creationTimestamp: "2026-07-16T17:46:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:32Z" name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb.18c2d6448d4c7739 namespace: kserve-ci-e2e-test resourceVersion: "15339" uid: 66601f6b-51cd-4667-8846-81f048e8bad4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 186e8cfd-e44b-4914-8c08-6c598764659b kind: Event lastTimestamp: "2026-07-16T17:46:32Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-16T17:46:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:32Z" name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb.18c2d6449046374e namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: 3f30cfc8-4394-4ad9-ac88-1a9d440f032e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 186e8cfd-e44b-4914-8c08-6c598764659b kind: Event lastTimestamp: "2026-07-16T17:46:32Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-16T17:46:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:32Z" name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb.18c2d64490d5a295 namespace: kserve-ci-e2e-test resourceVersion: "15341" uid: 005929ba-1ada-49f7-90fe-2b26adc3b336 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 186e8cfd-e44b-4914-8c08-6c598764659b kind: Event lastTimestamp: "2026-07-16T17:46:36Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-16T17:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:36Z" name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb.18c2d6459ab78d4e namespace: kserve-ci-e2e-test resourceVersion: "15389" uid: b2b90d14-44e4-4e7d-bf62-007059169f96 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 186e8cfd-e44b-4914-8c08-6c598764659b kind: Event lastTimestamp: "2026-07-16T17:46:56Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.943s (19.943s including waiting). Image size: 1306969339 bytes.' metadata: creationTimestamp: "2026-07-16T17:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:56Z" name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb.18c2d64a3f73beef namespace: kserve-ci-e2e-test resourceVersion: "15543" uid: 96a9c43d-3b8c-4bf5-9fa1-ffd1b2789d15 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 186e8cfd-e44b-4914-8c08-6c598764659b kind: Event lastTimestamp: "2026-07-16T17:46:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:56Z" name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb.18c2d64a42fee04f namespace: kserve-ci-e2e-test resourceVersion: "15547" uid: e8e326dc-cead-4d3d-91e4-be484ea427f7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 186e8cfd-e44b-4914-8c08-6c598764659b kind: Event lastTimestamp: "2026-07-16T17:46:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:56Z" name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb.18c2d64a43611363 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: cd39e3d1-e0ea-498c-a121-0471f76b9183 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 186e8cfd-e44b-4914-8c08-6c598764659b kind: Event lastTimestamp: "2026-07-16T17:46:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:56Z" name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb.18c2d64a436ce675 namespace: kserve-ci-e2e-test resourceVersion: "15549" uid: 5898db7b-f744-4c00-9f77-90222af13593 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 186e8cfd-e44b-4914-8c08-6c598764659b kind: Event lastTimestamp: "2026-07-16T17:46:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:56Z" name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb.18c2d64a462c971c namespace: kserve-ci-e2e-test resourceVersion: "15550" uid: aab63ae8-be97-4351-8dee-64b283df00fa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 186e8cfd-e44b-4914-8c08-6c598764659b kind: Event lastTimestamp: "2026-07-16T17:46:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:46:56Z" name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb.18c2d64a4694a5a6 namespace: kserve-ci-e2e-test resourceVersion: "15551" uid: 48f9eefc-b3a9-46b7-b14c-96ce9ba697ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-16T17:46:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 186e8cfd-e44b-4914-8c08-6c598764659b kind: Event lastTimestamp: "2026-07-16T17:48:23Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:46:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:23Z" name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb.18c2d64a8578aa09 namespace: kserve-ci-e2e-test resourceVersion: "16397" uid: 6c8bf321-07eb-4777-8eb1-44ffd3f7b0d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 186e8cfd-e44b-4914-8c08-6c598764659b kind: Event lastTimestamp: "2026-07-16T17:48:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:21Z" name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb.18c2d65e0c8d967f namespace: kserve-ci-e2e-test resourceVersion: "16347" uid: 6389f496-e992-40ca-a1da-da5924efe0be reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 186e8cfd-e44b-4914-8c08-6c598764659b kind: Event lastTimestamp: "2026-07-16T17:48:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:21Z" name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb.18c2d65e0c8e12ac namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: dd03fa00-d47d-4971-a17a-2b362febf35b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 186e8cfd-e44b-4914-8c08-6c598764659b kind: Event lastTimestamp: "2026-07-16T17:48:23Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:48:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:23Z" name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb.18c2d65e8bb41cf1 namespace: kserve-ci-e2e-test resourceVersion: "16396" uid: 68aba1e7-76e7-49d7-b9e0-3e34ad0d9066 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f namespace: kserve-ci-e2e-test resourceVersion: "15286" uid: f6ec2f8d-6462-4319-a4a2-b5e9668d9b2e kind: Event lastTimestamp: "2026-07-16T17:46:31Z" message: 'Created pod: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f-fnmjb' metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:31Z" name: isvc-xgboost-graph-raw-1404c-predictor-857fcf848f.18c2d6444dc4283c namespace: kserve-ci-e2e-test resourceVersion: "15297" uid: 629d87d0-4684-4422-9245-df99be051cb6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-1404c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15281" uid: 8eaa60c8-4ca6-46d4-8e1c-9b9403b6e32d kind: Event lastTimestamp: "2026-07-16T17:46:31Z" message: Scaled up replica set isvc-xgboost-graph-raw-1404c-predictor-857fcf848f from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:46:31Z" name: isvc-xgboost-graph-raw-1404c-predictor.18c2d6444acd9740 namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: ab605507-e73e-4148-a6f4-f75119c72f53 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:46:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-1404c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15285" uid: a4718a56-9cb3-429b-8eb0-406ec186d2f9 kind: Event lastTimestamp: "2026-07-16T17:47:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:01Z" name: isvc-xgboost-graph-raw-1404c-predictor.18c2d647c9643387 namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: 8739c817-3eab-45e7-84b9-093b8153f689 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:46:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-1404c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15285" uid: a4718a56-9cb3-429b-8eb0-406ec186d2f9 kind: Event lastTimestamp: "2026-07-16T17:47:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:47:01Z" name: isvc-xgboost-graph-raw-1404c-predictor.18c2d647c9d35e2e namespace: kserve-ci-e2e-test resourceVersion: "15592" uid: 407e2914-6c52-4b02-8222-8432633471ab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T17:47:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-1404c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15473" uid: a4718a56-9cb3-429b-8eb0-406ec186d2f9 kind: Event lastTimestamp: "2026-07-16T17:48:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:16Z" name: isvc-xgboost-graph-raw-1404c-predictor.18c2d64ec7a3dff5 namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: d378f2d2-01fd-442a-a9c2-75fefced5f7f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T17:47:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-1404c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15473" uid: a4718a56-9cb3-429b-8eb0-406ec186d2f9 kind: Event lastTimestamp: "2026-07-16T17:48:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:16Z" name: isvc-xgboost-graph-raw-1404c-predictor.18c2d64ec812b646 namespace: kserve-ci-e2e-test resourceVersion: "16193" uid: 857c7358-474f-4efb-a8c4-7241f55e28f6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-16T17:46:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-1404c namespace: kserve-ci-e2e-test resourceVersion: "15264" uid: 3424e879-a402-4e33-9306-03a64d36f4d7 kind: Event lastTimestamp: "2026-07-16T17:48:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:48:04Z" name: isvc-xgboost-graph-raw-1404c.18c2d6444aff523b namespace: kserve-ci-e2e-test resourceVersion: "16070" uid: 3863ee21-22fd-4ae0-8fec-361801ab4983 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:46:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-1404c namespace: kserve-ci-e2e-test resourceVersion: "15264" uid: 3424e879-a402-4e33-9306-03a64d36f4d7 kind: Event lastTimestamp: "2026-07-16T17:46:31Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-1404c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-1404c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:46:31Z" name: isvc-xgboost-graph-raw-1404c.18c2d6444b66159c namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: b654e8ea-9655-4004-be33-3ba114a6f99e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-1404c namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: 3424e879-a402-4e33-9306-03a64d36f4d7 kind: Event lastTimestamp: "2026-07-16T17:48:04Z" message: InferenceService [isvc-xgboost-graph-raw-1404c] is Ready metadata: creationTimestamp: "2026-07-16T17:48:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:48:04Z" name: isvc-xgboost-graph-raw-1404c.18c2d659e9813a7d namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 2e3aae5c-c388-4881-b234-9d608821bf0f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:48:21.722608Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: 7fb5e365-caf4-4df8-ba1e-13d9a1adbd69 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl to ip-10-0-130-5.ec2.internal metadata: creationTimestamp: "2026-07-16T17:48:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:48:21Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl.18c2d65e06dde38a namespace: kserve-ci-e2e-test resourceVersion: "16323" uid: 5759cfff-0d53-4beb-b113-009602321551 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-699544b6dc-lkd89 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:22Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 7fb5e365-caf4-4df8-ba1e-13d9a1adbd69 kind: Event lastTimestamp: "2026-07-16T17:48:22Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:48:22Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl.18c2d65e20d115b9 namespace: kserve-ci-e2e-test resourceVersion: "16365" uid: 917b343c-27ad-48cf-a614-65b692e05a52 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 7fb5e365-caf4-4df8-ba1e-13d9a1adbd69 kind: Event lastTimestamp: "2026-07-16T17:48:22Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:86207d1cba5aecf8e7b5bca617ed35061e54e289c7778b383f47cc8d53214f1a" already present on machine metadata: creationTimestamp: "2026-07-16T17:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:22Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl.18c2d65e2403ca77 namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: fa858be4-e22c-44a1-ad09-3ef8f89ce476 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-5.ec2.internal source: component: kubelet host: ip-10-0-130-5.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 7fb5e365-caf4-4df8-ba1e-13d9a1adbd69 kind: Event lastTimestamp: "2026-07-16T17:48:22Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-16T17:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:22Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl.18c2d65e26a8dfa1 namespace: kserve-ci-e2e-test resourceVersion: "16370" uid: bd7ba18c-7392-4bee-9dcc-1d693d1a47c0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-5.ec2.internal source: component: kubelet host: ip-10-0-130-5.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 7fb5e365-caf4-4df8-ba1e-13d9a1adbd69 kind: Event lastTimestamp: "2026-07-16T17:48:22Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-16T17:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:22Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl.18c2d65e271d76d3 namespace: kserve-ci-e2e-test resourceVersion: "16371" uid: b489c438-a67a-440e-89eb-bb65bd34ef63 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-5.ec2.internal source: component: kubelet host: ip-10-0-130-5.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 7fb5e365-caf4-4df8-ba1e-13d9a1adbd69 kind: Event lastTimestamp: "2026-07-16T17:48:26Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-16T17:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:26Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl.18c2d65f3383d39b namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: 01d5aa18-4005-4975-8cf8-96af0e9b7372 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-5.ec2.internal source: component: kubelet host: ip-10-0-130-5.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 7fb5e365-caf4-4df8-ba1e-13d9a1adbd69 kind: Event lastTimestamp: "2026-07-16T17:48:47Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 20.86s (20.86s including waiting). Image size: 1306969339 bytes.' metadata: creationTimestamp: "2026-07-16T17:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:47Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl.18c2d6640ee33709 namespace: kserve-ci-e2e-test resourceVersion: "16597" uid: bcdab9e5-1c1c-461a-adc6-5febe89ab17c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-5.ec2.internal source: component: kubelet host: ip-10-0-130-5.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 7fb5e365-caf4-4df8-ba1e-13d9a1adbd69 kind: Event lastTimestamp: "2026-07-16T17:48:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:47Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl.18c2d664123b9cca namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: 04227250-0549-4b46-bbf5-062356441fd9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-5.ec2.internal source: component: kubelet host: ip-10-0-130-5.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 7fb5e365-caf4-4df8-ba1e-13d9a1adbd69 kind: Event lastTimestamp: "2026-07-16T17:48:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:47Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl.18c2d66412dbb41b namespace: kserve-ci-e2e-test resourceVersion: "16599" uid: 395c795c-9c1b-43b5-b035-7eab55f3ffa9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-5.ec2.internal source: component: kubelet host: ip-10-0-130-5.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 7fb5e365-caf4-4df8-ba1e-13d9a1adbd69 kind: Event lastTimestamp: "2026-07-16T17:48:47Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-16T17:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:47Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl.18c2d66412e927d2 namespace: kserve-ci-e2e-test resourceVersion: "16600" uid: 5058ed6a-a206-4f53-9563-54bd878219a3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-5.ec2.internal source: component: kubelet host: ip-10-0-130-5.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 7fb5e365-caf4-4df8-ba1e-13d9a1adbd69 kind: Event lastTimestamp: "2026-07-16T17:48:50Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.858s (2.858s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-16T17:48:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:50Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl.18c2d664bd48e3bb namespace: kserve-ci-e2e-test resourceVersion: "16621" uid: 372f0303-7f3a-43bc-a7fc-595f675cc016 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-5.ec2.internal source: component: kubelet host: ip-10-0-130-5.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 7fb5e365-caf4-4df8-ba1e-13d9a1adbd69 kind: Event lastTimestamp: "2026-07-16T17:48:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:48:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:50Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl.18c2d664c38ccaa9 namespace: kserve-ci-e2e-test resourceVersion: "16624" uid: 023d64a8-34f7-4d00-8260-7beef48f9d76 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-5.ec2.internal source: component: kubelet host: ip-10-0-130-5.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 7fb5e365-caf4-4df8-ba1e-13d9a1adbd69 kind: Event lastTimestamp: "2026-07-16T17:48:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:48:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:50Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl.18c2d664c4f8cf2c namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 3a0588a5-1610-46f1-bff7-19d9bb69f56f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-5.ec2.internal source: component: kubelet host: ip-10-0-130-5.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-16T17:48:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 7fb5e365-caf4-4df8-ba1e-13d9a1adbd69 kind: Event lastTimestamp: "2026-07-16T17:49:47Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:48:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:49:47Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl.18c2d6650d546a5b namespace: kserve-ci-e2e-test resourceVersion: "17068" uid: edad7986-bc36-4435-9ea4-bb91a301c07c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-5.ec2.internal source: component: kubelet host: ip-10-0-130-5.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 7fb5e365-caf4-4df8-ba1e-13d9a1adbd69 kind: Event lastTimestamp: "2026-07-16T17:50:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:12Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl.18c2d677b3e6b342 namespace: kserve-ci-e2e-test resourceVersion: "17373" uid: 63d36925-bf58-41ad-a00b-9ff8371ec747 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-5.ec2.internal source: component: kubelet host: ip-10-0-130-5.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 7fb5e365-caf4-4df8-ba1e-13d9a1adbd69 kind: Event lastTimestamp: "2026-07-16T17:50:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:12Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl.18c2d677b3e7426c namespace: kserve-ci-e2e-test resourceVersion: "17374" uid: 896c2bf3-01a1-4792-9245-e9fa29a545e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-5.ec2.internal source: component: kubelet host: ip-10-0-130-5.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 7fb5e365-caf4-4df8-ba1e-13d9a1adbd69 kind: Event lastTimestamp: "2026-07-16T17:50:12Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:12Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl.18c2d677e999a67c namespace: kserve-ci-e2e-test resourceVersion: "17386" uid: 2594513f-a28f-4498-ac39-6a0af3da18d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-5.ec2.internal source: component: kubelet host: ip-10-0-130-5.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5 namespace: kserve-ci-e2e-test resourceVersion: "16311" uid: 22ad3960-cf2f-4bd3-a6eb-2c824bdb00ca kind: Event lastTimestamp: "2026-07-16T17:48:21Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5-nmszl' metadata: creationTimestamp: "2026-07-16T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:21Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5.18c2d65e063966f4 namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: 00dbafce-c8fc-4c8e-8a29-01551b09a0cd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-57df2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16308" uid: 630264be-6448-44bb-a2eb-973e8453ae28 kind: Event lastTimestamp: "2026-07-16T17:48:21Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-57df2-predictor-58f5fb86d5 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:21Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor.18c2d65dfff1b6e8 namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: cfacfd5d-eb84-4519-834d-4d87eef9cc9b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T17:48:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-57df2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: b730ebad-de4d-4252-afa2-03f78b125bac kind: Event lastTimestamp: "2026-07-16T17:49:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:49:21Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor.18c2d6617e29e8a0 namespace: kserve-ci-e2e-test resourceVersion: "16871" uid: c97a0425-d2c2-486f-b455-f58b2c2c2f49 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T17:48:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-57df2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: b730ebad-de4d-4252-afa2-03f78b125bac kind: Event lastTimestamp: "2026-07-16T17:49:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:49:21Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor.18c2d6617ea63d17 namespace: kserve-ci-e2e-test resourceVersion: "16872" uid: c1820cac-bcf4-4da8-bf5a-ff1ef653d5e1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:49:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-57df2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16521" uid: b730ebad-de4d-4252-afa2-03f78b125bac kind: Event lastTimestamp: "2026-07-16T17:50:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:50:06Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor.18c2d66f79f11422 namespace: kserve-ci-e2e-test resourceVersion: "17245" uid: aef2db99-3263-4fac-bca1-bc15f056e609 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:49:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-57df2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16521" uid: b730ebad-de4d-4252-afa2-03f78b125bac kind: Event lastTimestamp: "2026-07-16T17:50:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:50:06Z" name: isvc-xgboost-graph-raw-hpa-57df2-predictor.18c2d66f7a60b2ce namespace: kserve-ci-e2e-test resourceVersion: "17246" uid: b67994b2-3f0e-46ba-b0dd-2393968192d4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-16T17:48:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-57df2 namespace: kserve-ci-e2e-test resourceVersion: "16289" uid: 2319f0ce-6888-4fd7-824f-a3958bc98bb4 kind: Event lastTimestamp: "2026-07-16T17:49:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:49:58Z" name: isvc-xgboost-graph-raw-hpa-57df2.18c2d65dfff958c9 namespace: kserve-ci-e2e-test resourceVersion: "17143" uid: d545da96-1e0f-41e1-b44c-2f3ed7c32eb4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-57df2 namespace: kserve-ci-e2e-test resourceVersion: "16289" uid: 2319f0ce-6888-4fd7-824f-a3958bc98bb4 kind: Event lastTimestamp: "2026-07-16T17:48:21Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-57df2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-57df2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:48:21Z" name: isvc-xgboost-graph-raw-hpa-57df2.18c2d65e018766fd namespace: kserve-ci-e2e-test resourceVersion: "16316" uid: 6ffb14eb-7f5d-4e70-92b9-66ed55a994d7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:49:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-57df2 namespace: kserve-ci-e2e-test resourceVersion: "17141" uid: 2319f0ce-6888-4fd7-824f-a3958bc98bb4 kind: Event lastTimestamp: "2026-07-16T17:49:58Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-57df2] is Ready metadata: creationTimestamp: "2026-07-16T17:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:49:58Z" name: isvc-xgboost-graph-raw-hpa-57df2.18c2d6747177d941 namespace: kserve-ci-e2e-test resourceVersion: "17142" uid: a8bda6b1-3a10-4dac-84be-80b10d05a0d6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:50:11.867510Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c namespace: kserve-ci-e2e-test resourceVersion: "17346" uid: 15bc05f5-8881-495b-8510-ac0076c95f6f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c to ip-10-0-132-218.ec2.internal metadata: creationTimestamp: "2026-07-16T17:50:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:50:11Z" name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c.18c2d677ac03c73f namespace: kserve-ci-e2e-test resourceVersion: "17352" uid: a04c6744-d7c0-4fe5-be80-1f327d70710f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-699544b6dc-lkd89 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:12Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 15bc05f5-8881-495b-8510-ac0076c95f6f kind: Event lastTimestamp: "2026-07-16T17:50:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-raw-c3a17-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T17:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:12Z" name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c.18c2d677b6762baf namespace: kserve-ci-e2e-test resourceVersion: "17376" uid: f1e22b80-43bd-4170-aeef-6fab41731b29 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:12Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c namespace: kserve-ci-e2e-test resourceVersion: "17356" uid: 15bc05f5-8881-495b-8510-ac0076c95f6f kind: Event lastTimestamp: "2026-07-16T17:50:12Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:50:12Z" name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c.18c2d677e9b8988b namespace: kserve-ci-e2e-test resourceVersion: "17387" uid: 1710ba9c-4e5b-425c-aeda-59d1104980f2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 15bc05f5-8881-495b-8510-ac0076c95f6f kind: Event lastTimestamp: "2026-07-16T17:50:12Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-16T17:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:12Z" name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c.18c2d677eb22dd21 namespace: kserve-ci-e2e-test resourceVersion: "17389" uid: 904c7800-8a71-4ae6-a33e-d746ee6dbdae reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 15bc05f5-8881-495b-8510-ac0076c95f6f kind: Event lastTimestamp: "2026-07-16T17:50:14Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.594s (1.594s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-16T17:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:14Z" name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c.18c2d6784a30af99 namespace: kserve-ci-e2e-test resourceVersion: "17404" uid: c9bb7084-3079-4be9-8e9f-76c86a3e513f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 15bc05f5-8881-495b-8510-ac0076c95f6f kind: Event lastTimestamp: "2026-07-16T17:50:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:14Z" name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c.18c2d6784d476f4d namespace: kserve-ci-e2e-test resourceVersion: "17405" uid: b217667a-92cb-430b-80b5-495884d7478e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 15bc05f5-8881-495b-8510-ac0076c95f6f kind: Event lastTimestamp: "2026-07-16T17:50:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:14Z" name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c.18c2d6784dbf468a namespace: kserve-ci-e2e-test resourceVersion: "17406" uid: d573cf8b-6a71-448d-88ff-69d1b68edfa9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 15bc05f5-8881-495b-8510-ac0076c95f6f kind: Event lastTimestamp: "2026-07-16T17:50:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:14Z" name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c.18c2d6784dcba9ff namespace: kserve-ci-e2e-test resourceVersion: "17408" uid: ecc20507-18b6-4a0e-ab71-f1568653c3e3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 15bc05f5-8881-495b-8510-ac0076c95f6f kind: Event lastTimestamp: "2026-07-16T17:50:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:14Z" name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c.18c2d67850d42090 namespace: kserve-ci-e2e-test resourceVersion: "17410" uid: 05872eac-9cf4-4402-88fe-4a8d38acec9d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 15bc05f5-8881-495b-8510-ac0076c95f6f kind: Event lastTimestamp: "2026-07-16T17:50:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:14Z" name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c.18c2d67851459853 namespace: kserve-ci-e2e-test resourceVersion: "17411" uid: 4af502e1-de67-4c45-ad65-2e4124ee61e9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 15bc05f5-8881-495b-8510-ac0076c95f6f kind: Event lastTimestamp: "2026-07-16T17:52:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:52:07Z" name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c.18c2d6928d70b4e4 namespace: kserve-ci-e2e-test resourceVersion: "18386" uid: 0d867cca-7c7b-494a-9864-849d69695512 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 15bc05f5-8881-495b-8510-ac0076c95f6f kind: Event lastTimestamp: "2026-07-16T17:52:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:52:07Z" name: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c.18c2d6928d72156a namespace: kserve-ci-e2e-test resourceVersion: "18388" uid: 53fcbecf-e1c6-4a86-999b-80f0a7674293 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-c3a17-predictor-6788446ffc namespace: kserve-ci-e2e-test resourceVersion: "17332" uid: 2bddccf4-dae3-4d0c-a01a-3225ddf28136 kind: Event lastTimestamp: "2026-07-16T17:50:11Z" message: 'Created pod: message-dumper-raw-c3a17-predictor-6788446ffc-h9n8c' metadata: creationTimestamp: "2026-07-16T17:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:50:11Z" name: message-dumper-raw-c3a17-predictor-6788446ffc.18c2d677ab5ff907 namespace: kserve-ci-e2e-test resourceVersion: "17348" uid: c43e744c-7642-4fe6-8c6c-ef9f3b46804f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-c3a17-predictor namespace: kserve-ci-e2e-test resourceVersion: "17323" uid: e216dc5d-d2b1-42c3-81ca-a7bfcbbdd939 kind: Event lastTimestamp: "2026-07-16T17:50:11Z" message: Scaled up replica set message-dumper-raw-c3a17-predictor-6788446ffc from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:50:11Z" name: message-dumper-raw-c3a17-predictor.18c2d677a7b8ea32 namespace: kserve-ci-e2e-test resourceVersion: "17334" uid: 9a670efa-6f5d-4ecf-a53f-8a6ea84effa8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T17:50:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-c3a17-predictor namespace: kserve-ci-e2e-test resourceVersion: "17329" uid: c218d1a3-ad43-4a0f-897e-6d7be01ec03f kind: Event lastTimestamp: "2026-07-16T17:51:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:51:11Z" name: message-dumper-raw-c3a17-predictor.18c2d67b2716fbd2 namespace: kserve-ci-e2e-test resourceVersion: "17910" uid: 205861c1-e618-4e68-9070-947934e5f74c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T17:50:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-c3a17-predictor namespace: kserve-ci-e2e-test resourceVersion: "17329" uid: c218d1a3-ad43-4a0f-897e-6d7be01ec03f kind: Event lastTimestamp: "2026-07-16T17:51:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:51:11Z" name: message-dumper-raw-c3a17-predictor.18c2d67b2791069b namespace: kserve-ci-e2e-test resourceVersion: "17911" uid: 22839d64-65dd-4056-8b61-412873dc9c17 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-16T17:50:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-c3a17 namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: ea110fbd-1eca-4731-a4dc-48bade6c0871 kind: Event lastTimestamp: "2026-07-16T17:50:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:50:23Z" name: message-dumper-raw-c3a17.18c2d677a7ade91c namespace: kserve-ci-e2e-test resourceVersion: "17483" uid: 430ca172-4dcb-42c7-8278-bf794968e871 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-c3a17 namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: ea110fbd-1eca-4731-a4dc-48bade6c0871 kind: Event lastTimestamp: "2026-07-16T17:50:11Z" message: 'Failed to update status for InferenceService "message-dumper-raw-c3a17": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-c3a17": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:50:11Z" name: message-dumper-raw-c3a17.18c2d677a845c24e namespace: kserve-ci-e2e-test resourceVersion: "17338" uid: 802e1bdc-4623-44c2-b18b-5e13524f96b6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-c3a17 namespace: kserve-ci-e2e-test resourceVersion: "17479" uid: ea110fbd-1eca-4731-a4dc-48bade6c0871 kind: Event lastTimestamp: "2026-07-16T17:50:22Z" message: InferenceService [message-dumper-raw-c3a17] is Ready metadata: creationTimestamp: "2026-07-16T17:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:50:22Z" name: message-dumper-raw-c3a17.18c2d67a4147be2a namespace: kserve-ci-e2e-test resourceVersion: "17480" uid: 0e063e9a-25d3-4ef6-9bf0-6b32c333b982 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:48:11.168687Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-1404c-5cfdb5fbdb-rvh6w namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: 853fedf2-3f18-4cb6-ab71-0751dd15608d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-1404c-5cfdb5fbdb-rvh6w to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-07-16T17:48:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:48:11Z" name: model-chainer-raw-1404c-5cfdb5fbdb-rvh6w.18c2d65b91cdd396 namespace: kserve-ci-e2e-test resourceVersion: "16132" uid: 6684644b-afbd-4844-a327-5d05cada36e0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-699544b6dc-lkd89 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:11Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-1404c-5cfdb5fbdb-rvh6w namespace: kserve-ci-e2e-test resourceVersion: "16129" uid: 853fedf2-3f18-4cb6-ab71-0751dd15608d kind: Event lastTimestamp: "2026-07-16T17:48:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-1404c-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T17:48:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:11Z" name: model-chainer-raw-1404c-5cfdb5fbdb-rvh6w.18c2d65ba1619a08 namespace: kserve-ci-e2e-test resourceVersion: "16144" uid: b2066fe2-6d0b-44ca-9146-2b8e90b5ec8a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:12Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-1404c-5cfdb5fbdb-rvh6w namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: 853fedf2-3f18-4cb6-ab71-0751dd15608d kind: Event lastTimestamp: "2026-07-16T17:48:12Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:48:12Z" name: model-chainer-raw-1404c-5cfdb5fbdb-rvh6w.18c2d65bcfc4f69b namespace: kserve-ci-e2e-test resourceVersion: "16153" uid: f1e0460f-6a70-4b14-b10f-e5d48e13328f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-1404c} kind: Pod name: model-chainer-raw-1404c-5cfdb5fbdb-rvh6w namespace: kserve-ci-e2e-test resourceVersion: "16129" uid: 853fedf2-3f18-4cb6-ab71-0751dd15608d kind: Event lastTimestamp: "2026-07-16T17:48:12Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:4979cb407a123d624c47a64b3f1979504d753f7cd80622e43a8523addc69e145" metadata: creationTimestamp: "2026-07-16T17:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:12Z" name: model-chainer-raw-1404c-5cfdb5fbdb-rvh6w.18c2d65bd0f2c08f namespace: kserve-ci-e2e-test resourceVersion: "16155" uid: abf541c9-4a0f-457d-b19a-9b1b5885e58f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-1404c} kind: Pod name: model-chainer-raw-1404c-5cfdb5fbdb-rvh6w namespace: kserve-ci-e2e-test resourceVersion: "16129" uid: 853fedf2-3f18-4cb6-ab71-0751dd15608d kind: Event lastTimestamp: "2026-07-16T17:48:14Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:4979cb407a123d624c47a64b3f1979504d753f7cd80622e43a8523addc69e145" in 2.088s (2.088s including waiting). Image size: 218081401 bytes.' metadata: creationTimestamp: "2026-07-16T17:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:14Z" name: model-chainer-raw-1404c-5cfdb5fbdb-rvh6w.18c2d65c4d70076a namespace: kserve-ci-e2e-test resourceVersion: "16168" uid: 957cd1ef-4f57-4751-b168-10343f5c28b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-1404c} kind: Pod name: model-chainer-raw-1404c-5cfdb5fbdb-rvh6w namespace: kserve-ci-e2e-test resourceVersion: "16129" uid: 853fedf2-3f18-4cb6-ab71-0751dd15608d kind: Event lastTimestamp: "2026-07-16T17:48:14Z" message: 'Created container: model-chainer-raw-1404c' metadata: creationTimestamp: "2026-07-16T17:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:14Z" name: model-chainer-raw-1404c-5cfdb5fbdb-rvh6w.18c2d65c50890506 namespace: kserve-ci-e2e-test resourceVersion: "16169" uid: 99610242-bbcc-49af-a156-c0fe761643d4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-1404c} kind: Pod name: model-chainer-raw-1404c-5cfdb5fbdb-rvh6w namespace: kserve-ci-e2e-test resourceVersion: "16129" uid: 853fedf2-3f18-4cb6-ab71-0751dd15608d kind: Event lastTimestamp: "2026-07-16T17:48:14Z" message: Started container model-chainer-raw-1404c metadata: creationTimestamp: "2026-07-16T17:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:14Z" name: model-chainer-raw-1404c-5cfdb5fbdb-rvh6w.18c2d65c50f773dc namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 4ab2f424-77e6-4194-9100-43a407910c8a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-1404c} kind: Pod name: model-chainer-raw-1404c-5cfdb5fbdb-rvh6w namespace: kserve-ci-e2e-test resourceVersion: "16129" uid: 853fedf2-3f18-4cb6-ab71-0751dd15608d kind: Event lastTimestamp: "2026-07-16T17:48:21Z" message: Stopping container model-chainer-raw-1404c metadata: creationTimestamp: "2026-07-16T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:21Z" name: model-chainer-raw-1404c-5cfdb5fbdb-rvh6w.18c2d65de9095ac8 namespace: kserve-ci-e2e-test resourceVersion: "16245" uid: 02931b2b-9802-4605-b27e-0c61890d9cf2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:48:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-1404c} kind: Pod name: model-chainer-raw-1404c-5cfdb5fbdb-rvh6w namespace: kserve-ci-e2e-test resourceVersion: "16129" uid: 853fedf2-3f18-4cb6-ab71-0751dd15608d kind: Event lastTimestamp: "2026-07-16T17:48:51Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T17:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:48:51Z" name: model-chainer-raw-1404c-5cfdb5fbdb-rvh6w.18c2d65f0928c4c5 namespace: kserve-ci-e2e-test resourceVersion: "16631" uid: 8b84105e-05dd-428f-be65-6fc8be1ef219 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-1404c-5cfdb5fbdb namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: 277cdba3-694d-4244-8937-292edce7ebbb kind: Event lastTimestamp: "2026-07-16T17:48:11Z" message: 'Created pod: model-chainer-raw-1404c-5cfdb5fbdb-rvh6w' metadata: creationTimestamp: "2026-07-16T17:48:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:11Z" name: model-chainer-raw-1404c-5cfdb5fbdb.18c2d65b913e02f1 namespace: kserve-ci-e2e-test resourceVersion: "16128" uid: 6098aa78-905a-48cb-b09f-4bac94d424e5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-1404c namespace: kserve-ci-e2e-test resourceVersion: "16122" uid: f27220ef-20e3-44a9-a866-5d9ace7ea481 kind: Event lastTimestamp: "2026-07-16T17:48:11Z" message: Scaled up replica set model-chainer-raw-1404c-5cfdb5fbdb from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:48:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:48:11Z" name: model-chainer-raw-1404c.18c2d65b9040c52c namespace: kserve-ci-e2e-test resourceVersion: "16125" uid: 9e0d5980-09dd-47fd-9722-d16d02a484ed reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:48:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-1404c namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: 557e3181-d1fe-4d58-85f4-1d59cbf880fe kind: Event lastTimestamp: "2026-07-16T17:48:11Z" message: InferenceGraph [model-chainer-raw-1404c] is Ready metadata: creationTimestamp: "2026-07-16T17:48:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:48:11Z" name: model-chainer-raw-1404c.18c2d65b9618bb4c namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: d2c0202f-0a9e-4827-9f92-6ba2dc4a465f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:50:01.577475Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-57df2-6d8bb98f86-5n92b namespace: kserve-ci-e2e-test resourceVersion: "17184" uid: a2d28e41-984e-440b-a627-67bfc5772aab kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-57df2-6d8bb98f86-5n92b to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-07-16T17:50:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:50:01Z" name: model-chainer-raw-hpa-57df2-6d8bb98f86-5n92b.18c2d67546ae41f6 namespace: kserve-ci-e2e-test resourceVersion: "17189" uid: 31e2d414-65dc-4f52-9b79-9dcedc74e80a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-699544b6dc-lkd89 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:01Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-57df2-6d8bb98f86-5n92b namespace: kserve-ci-e2e-test resourceVersion: "17187" uid: a2d28e41-984e-440b-a627-67bfc5772aab kind: Event lastTimestamp: "2026-07-16T17:50:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-57df2-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T17:50:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:01Z" name: model-chainer-raw-hpa-57df2-6d8bb98f86-5n92b.18c2d67552330260 namespace: kserve-ci-e2e-test resourceVersion: "17201" uid: 202a9808-bc98-431b-b490-d37de1e0ac58 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:02Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-57df2-6d8bb98f86-5n92b namespace: kserve-ci-e2e-test resourceVersion: "17193" uid: a2d28e41-984e-440b-a627-67bfc5772aab kind: Event lastTimestamp: "2026-07-16T17:50:02Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:50:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:50:02Z" name: model-chainer-raw-hpa-57df2-6d8bb98f86-5n92b.18c2d675836fdcc4 namespace: kserve-ci-e2e-test resourceVersion: "17209" uid: a3decb73-61fc-4dc0-b2d6-bdd5c017dc68 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-57df2} kind: Pod name: model-chainer-raw-hpa-57df2-6d8bb98f86-5n92b namespace: kserve-ci-e2e-test resourceVersion: "17187" uid: a2d28e41-984e-440b-a627-67bfc5772aab kind: Event lastTimestamp: "2026-07-16T17:50:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4979cb407a123d624c47a64b3f1979504d753f7cd80622e43a8523addc69e145" already present on machine metadata: creationTimestamp: "2026-07-16T17:50:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:02Z" name: model-chainer-raw-hpa-57df2-6d8bb98f86-5n92b.18c2d67584bae565 namespace: kserve-ci-e2e-test resourceVersion: "17211" uid: 05097e80-13d2-449d-b24f-fdf80d06f997 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-57df2} kind: Pod name: model-chainer-raw-hpa-57df2-6d8bb98f86-5n92b namespace: kserve-ci-e2e-test resourceVersion: "17187" uid: a2d28e41-984e-440b-a627-67bfc5772aab kind: Event lastTimestamp: "2026-07-16T17:50:02Z" message: 'Created container: model-chainer-raw-hpa-57df2' metadata: creationTimestamp: "2026-07-16T17:50:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:02Z" name: model-chainer-raw-hpa-57df2-6d8bb98f86-5n92b.18c2d67587bf53f9 namespace: kserve-ci-e2e-test resourceVersion: "17212" uid: 566fe53a-154d-4061-ba67-0d36d01da7f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-57df2} kind: Pod name: model-chainer-raw-hpa-57df2-6d8bb98f86-5n92b namespace: kserve-ci-e2e-test resourceVersion: "17187" uid: a2d28e41-984e-440b-a627-67bfc5772aab kind: Event lastTimestamp: "2026-07-16T17:50:02Z" message: Started container model-chainer-raw-hpa-57df2 metadata: creationTimestamp: "2026-07-16T17:50:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:02Z" name: model-chainer-raw-hpa-57df2-6d8bb98f86-5n92b.18c2d67588432c64 namespace: kserve-ci-e2e-test resourceVersion: "17213" uid: dfa50f77-efcb-4fb5-9e2e-7195033e3852 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-57df2} kind: Pod name: model-chainer-raw-hpa-57df2-6d8bb98f86-5n92b namespace: kserve-ci-e2e-test resourceVersion: "17187" uid: a2d28e41-984e-440b-a627-67bfc5772aab kind: Event lastTimestamp: "2026-07-16T17:50:11Z" message: Stopping container model-chainer-raw-hpa-57df2 metadata: creationTimestamp: "2026-07-16T17:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:11Z" name: model-chainer-raw-hpa-57df2-6d8bb98f86-5n92b.18c2d6779d223865 namespace: kserve-ci-e2e-test resourceVersion: "17303" uid: b2849673-99fa-4e44-875f-79471f7e2650 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T17:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-57df2} kind: Pod name: model-chainer-raw-hpa-57df2-6d8bb98f86-5n92b namespace: kserve-ci-e2e-test resourceVersion: "17187" uid: a2d28e41-984e-440b-a627-67bfc5772aab kind: Event lastTimestamp: "2026-07-16T17:50:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T17:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:50:39Z" name: model-chainer-raw-hpa-57df2-6d8bb98f86-5n92b.18c2d67843f6daa7 namespace: kserve-ci-e2e-test resourceVersion: "17675" uid: 9114f754-e66a-4623-94da-61d158c28f4c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-57df2-6d8bb98f86 namespace: kserve-ci-e2e-test resourceVersion: "17181" uid: c0a44a03-d93d-4acf-bdce-2e45e57108df kind: Event lastTimestamp: "2026-07-16T17:50:01Z" message: 'Created pod: model-chainer-raw-hpa-57df2-6d8bb98f86-5n92b' metadata: creationTimestamp: "2026-07-16T17:50:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:50:01Z" name: model-chainer-raw-hpa-57df2-6d8bb98f86.18c2d67545fde373 namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: 03ca9768-1602-49c4-8f1e-f264b5408923 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-57df2 namespace: kserve-ci-e2e-test resourceVersion: "17179" uid: e84f28b5-7325-4baa-b606-24547443d3e7 kind: Event lastTimestamp: "2026-07-16T17:50:01Z" message: Scaled up replica set model-chainer-raw-hpa-57df2-6d8bb98f86 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:50:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:50:01Z" name: model-chainer-raw-hpa-57df2.18c2d67544902ace namespace: kserve-ci-e2e-test resourceVersion: "17182" uid: c37ac1d5-4b42-4c3f-abbf-7788e5f8ef70 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:50:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-57df2 namespace: kserve-ci-e2e-test resourceVersion: "17197" uid: 6c6bc6c5-a141-4d5f-b5f6-c8a4e9bf2490 kind: Event lastTimestamp: "2026-07-16T17:50:01Z" message: InferenceGraph [model-chainer-raw-hpa-57df2] is Ready metadata: creationTimestamp: "2026-07-16T17:50:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:50:01Z" name: model-chainer-raw-hpa-57df2.18c2d6754b898764 namespace: kserve-ci-e2e-test resourceVersion: "17199" uid: 0a8be1bf-6e1f-4a4e-b341-b3ca8210edde reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:57:11.485638Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j namespace: kserve-ci-e2e-test resourceVersion: "20884" uid: 69e56822-1430-4da0-85d0-630c33325a07 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-afd96-predictor-8d6f8647b-7hf9j to ip-10-0-132-218.ec2.internal metadata: creationTimestamp: "2026-07-16T17:57:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:57:11Z" name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j.18c2d6d95f343f3c namespace: kserve-ci-e2e-test resourceVersion: "20891" uid: 9d2c5c5b-264e-4861-a5f9-5e3ac7f065af reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-699544b6dc-lkd89 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:11Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j namespace: kserve-ci-e2e-test resourceVersion: "20896" uid: 69e56822-1430-4da0-85d0-630c33325a07 kind: Event lastTimestamp: "2026-07-16T17:57:11Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:57:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:57:11Z" name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j.18c2d6d979b6e40e namespace: kserve-ci-e2e-test resourceVersion: "20901" uid: bac9078e-9b71-4475-9a1c-aafa3412e3e5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: 69e56822-1430-4da0-85d0-630c33325a07 kind: Event lastTimestamp: "2026-07-16T17:57:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:86207d1cba5aecf8e7b5bca617ed35061e54e289c7778b383f47cc8d53214f1a" already present on machine metadata: creationTimestamp: "2026-07-16T17:57:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:11Z" name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j.18c2d6d97b24469a namespace: kserve-ci-e2e-test resourceVersion: "20903" uid: 2ddec45a-11af-4241-afe2-e49f7550a856 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: 69e56822-1430-4da0-85d0-630c33325a07 kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:12Z" name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j.18c2d6d97e17e3b3 namespace: kserve-ci-e2e-test resourceVersion: "20904" uid: 4b87871d-558e-4c68-bcef-9b2ddd6ad4b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: 69e56822-1430-4da0-85d0-630c33325a07 kind: Event lastTimestamp: "2026-07-16T17:57:12Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-16T17:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:12Z" name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j.18c2d6d97e9464e4 namespace: kserve-ci-e2e-test resourceVersion: "20905" uid: ff9524ba-442f-4d02-880b-ba71d874967b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: 69e56822-1430-4da0-85d0-630c33325a07 kind: Event lastTimestamp: "2026-07-16T17:57:16Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T17:57:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:16Z" name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j.18c2d6da95e9e5c4 namespace: kserve-ci-e2e-test resourceVersion: "20943" uid: 0bfd3901-1ab7-446f-9045-41cbb2e844c7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: 69e56822-1430-4da0-85d0-630c33325a07 kind: Event lastTimestamp: "2026-07-16T17:57:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:57:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:16Z" name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j.18c2d6da9a2629c2 namespace: kserve-ci-e2e-test resourceVersion: "20947" uid: fd910b1a-1236-444c-86cf-782fffe499ca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: 69e56822-1430-4da0-85d0-630c33325a07 kind: Event lastTimestamp: "2026-07-16T17:57:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:57:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:16Z" name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j.18c2d6da9abf193a namespace: kserve-ci-e2e-test resourceVersion: "20948" uid: 8a65ca70-9fc0-4276-81a8-0c2dadba63b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: 69e56822-1430-4da0-85d0-630c33325a07 kind: Event lastTimestamp: "2026-07-16T17:57:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:57:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:16Z" name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j.18c2d6da9aca03fc namespace: kserve-ci-e2e-test resourceVersion: "20949" uid: 30547275-51be-4cbe-b7b2-f9fc31a9f89f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: 69e56822-1430-4da0-85d0-630c33325a07 kind: Event lastTimestamp: "2026-07-16T17:57:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:57:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:16Z" name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j.18c2d6da9de6256f namespace: kserve-ci-e2e-test resourceVersion: "20951" uid: 1863ae6c-ec87-4335-a523-3c287c217ddb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: 69e56822-1430-4da0-85d0-630c33325a07 kind: Event lastTimestamp: "2026-07-16T17:57:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:57:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:57:16Z" name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j.18c2d6da9e607125 namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: f5fe45e3-c6ec-4f22-bd77-9dcf4606d92b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-16T17:57:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: 69e56822-1430-4da0-85d0-630c33325a07 kind: Event lastTimestamp: "2026-07-16T17:58:44Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:44Z" name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j.18c2d6db0dcd7e6b namespace: kserve-ci-e2e-test resourceVersion: "21597" uid: 6debfb73-875f-479a-8884-ce29b8e63e64 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: 69e56822-1430-4da0-85d0-630c33325a07 kind: Event lastTimestamp: "2026-07-16T17:58:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T17:58:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:41Z" name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j.18c2d6ee5b060fa3 namespace: kserve-ci-e2e-test resourceVersion: "21572" uid: 97837ecf-6d6f-4bac-8df0-b8aa64ee9d08 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: 69e56822-1430-4da0-85d0-630c33325a07 kind: Event lastTimestamp: "2026-07-16T17:58:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:58:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:41Z" name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j.18c2d6ee5b073598 namespace: kserve-ci-e2e-test resourceVersion: "21573" uid: 1b29cf20-a247-49d1-8317-1152d1db7240 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: 69e56822-1430-4da0-85d0-630c33325a07 kind: Event lastTimestamp: "2026-07-16T17:58:44Z" message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp 10.133.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:44Z" name: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j.18c2d6ef141aaa0d namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: a8470921-3383-417d-8f98-31ccfd81a4ec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-afd96-predictor-8d6f8647b namespace: kserve-ci-e2e-test resourceVersion: "20875" uid: c3ceb0d6-7f19-4c96-ae6b-442a9efbb931 kind: Event lastTimestamp: "2026-07-16T17:57:11Z" message: 'Created pod: raw-sklearn-afd96-predictor-8d6f8647b-7hf9j' metadata: creationTimestamp: "2026-07-16T17:57:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:11Z" name: raw-sklearn-afd96-predictor-8d6f8647b.18c2d6d95ebc3c56 namespace: kserve-ci-e2e-test resourceVersion: "20885" uid: 318e8842-ead2-4d96-9801-19df7b0616c1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-afd96-predictor namespace: kserve-ci-e2e-test resourceVersion: "20873" uid: 260d705c-0cbe-4e39-9756-1a2a6d60d2c5 kind: Event lastTimestamp: "2026-07-16T17:57:11Z" message: Scaled up replica set raw-sklearn-afd96-predictor-8d6f8647b from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:57:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:11Z" name: raw-sklearn-afd96-predictor.18c2d6d95b3861c7 namespace: kserve-ci-e2e-test resourceVersion: "20876" uid: 1a82c831-5a10-4460-8f7d-3cbbc910fc35 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:57:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-afd96-predictor namespace: kserve-ci-e2e-test resourceVersion: "20874" uid: 6bae2083-9ca5-4ed6-8a1d-82bf9418222d kind: Event lastTimestamp: "2026-07-16T17:57:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:56Z" name: raw-sklearn-afd96-predictor.18c2d6dcdabe2864 namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: 98353eb2-e298-4b20-b5e1-b9c0c72a7328 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T17:57:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-afd96-predictor namespace: kserve-ci-e2e-test resourceVersion: "20874" uid: 6bae2083-9ca5-4ed6-8a1d-82bf9418222d kind: Event lastTimestamp: "2026-07-16T17:57:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:57:56Z" name: raw-sklearn-afd96-predictor.18c2d6dcdb36cb17 namespace: kserve-ci-e2e-test resourceVersion: "21234" uid: ec6cdba0-20de-408f-ac6e-669630013b61 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:58:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-afd96-predictor namespace: kserve-ci-e2e-test resourceVersion: "21024" uid: 6bae2083-9ca5-4ed6-8a1d-82bf9418222d kind: Event lastTimestamp: "2026-07-16T17:58:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:58:26Z" name: raw-sklearn-afd96-predictor.18c2d6e75b4edd10 namespace: kserve-ci-e2e-test resourceVersion: "21446" uid: 4352bb56-bd1a-4a74-b1c6-0bb6905baa63 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T17:58:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-afd96-predictor namespace: kserve-ci-e2e-test resourceVersion: "21024" uid: 6bae2083-9ca5-4ed6-8a1d-82bf9418222d kind: Event lastTimestamp: "2026-07-16T17:58:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:58:26Z" name: raw-sklearn-afd96-predictor.18c2d6e75c74a506 namespace: kserve-ci-e2e-test resourceVersion: "21447" uid: a179ba42-9277-4767-b74c-a400d90ba36e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-16T17:57:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-afd96 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 64a89508-5bae-4a23-ab90-161659c01956 kind: Event lastTimestamp: "2026-07-16T17:58:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:57:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:58:34Z" name: raw-sklearn-afd96.18c2d6d95b46beb4 namespace: kserve-ci-e2e-test resourceVersion: "21509" uid: 77ea2988-6e52-4200-9e8d-fe6c4b0cc921 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:57:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-afd96 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 64a89508-5bae-4a23-ab90-161659c01956 kind: Event lastTimestamp: "2026-07-16T17:57:11Z" message: 'Failed to update status for InferenceService "raw-sklearn-afd96": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-afd96": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:57:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:57:11Z" name: raw-sklearn-afd96.18c2d6d95bc7d47a namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: 074f4180-241a-425a-9eb5-0563a70b621c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-afd96 namespace: kserve-ci-e2e-test resourceVersion: "21506" uid: 64a89508-5bae-4a23-ab90-161659c01956 kind: Event lastTimestamp: "2026-07-16T17:58:34Z" message: InferenceService [raw-sklearn-afd96] is Ready metadata: creationTimestamp: "2026-07-16T17:58:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:58:34Z" name: raw-sklearn-afd96.18c2d6ecc701ca5f namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: c357ad8f-0860-4a40-8c38-08d22a62633e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T17:58:46.806004Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls namespace: kserve-ci-e2e-test resourceVersion: "21637" uid: 9d8630e6-683e-4fe9-acad-7602420ec371 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls to ip-10-0-132-218.ec2.internal metadata: creationTimestamp: "2026-07-16T17:58:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T17:58:46Z" name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls.18c2d6ef90bd98d3 namespace: kserve-ci-e2e-test resourceVersion: "21641" uid: 250e545c-bf41-498d-aedb-560da540ea10 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-699544b6dc-lkd89 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:47Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: 9d8630e6-683e-4fe9-acad-7602420ec371 kind: Event lastTimestamp: "2026-07-16T17:58:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-runtime-9159d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T17:58:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:47Z" name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls.18c2d6ef9cd4bd94 namespace: kserve-ci-e2e-test resourceVersion: "21650" uid: 64bbe169-0214-407e-b81b-2b2ab56b0bd1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:47Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls namespace: kserve-ci-e2e-test resourceVersion: "21645" uid: 9d8630e6-683e-4fe9-acad-7602420ec371 kind: Event lastTimestamp: "2026-07-16T17:58:47Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T17:58:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T17:58:47Z" name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls.18c2d6efce85a3de namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 3a7da1ed-0a1e-4c98-b8f9-95572905c31e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: 9d8630e6-683e-4fe9-acad-7602420ec371 kind: Event lastTimestamp: "2026-07-16T17:58:47Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:86207d1cba5aecf8e7b5bca617ed35061e54e289c7778b383f47cc8d53214f1a" already present on machine metadata: creationTimestamp: "2026-07-16T17:58:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:47Z" name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls.18c2d6efcfe7470e namespace: kserve-ci-e2e-test resourceVersion: "21663" uid: b7f0d859-0a0e-4627-b4ce-d0e91eb4ce49 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: 9d8630e6-683e-4fe9-acad-7602420ec371 kind: Event lastTimestamp: "2026-07-16T17:58:47Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-16T17:58:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:47Z" name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls.18c2d6efd2e13313 namespace: kserve-ci-e2e-test resourceVersion: "21664" uid: eadfd1ba-71c2-45ab-92d2-227f4b2fd478 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: 9d8630e6-683e-4fe9-acad-7602420ec371 kind: Event lastTimestamp: "2026-07-16T17:58:47Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-16T17:58:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:47Z" name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls.18c2d6efd3502bf3 namespace: kserve-ci-e2e-test resourceVersion: "21665" uid: d392cfa9-b884-4c0e-ae6b-b47238b84520 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: 9d8630e6-683e-4fe9-acad-7602420ec371 kind: Event lastTimestamp: "2026-07-16T17:58:52Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1744" already present on machine metadata: creationTimestamp: "2026-07-16T17:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:52Z" name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls.18c2d6f0cd444e33 namespace: kserve-ci-e2e-test resourceVersion: "21704" uid: e16ff793-aef9-45c5-8106-c379f4ff82fc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: 9d8630e6-683e-4fe9-acad-7602420ec371 kind: Event lastTimestamp: "2026-07-16T17:58:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T17:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:52Z" name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls.18c2d6f0d023149c namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: 6b93f9af-ed00-4155-9dd3-64d20e92e70e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: 9d8630e6-683e-4fe9-acad-7602420ec371 kind: Event lastTimestamp: "2026-07-16T17:58:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T17:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:52Z" name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls.18c2d6f0d09c24ec namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: c6ab8b77-a5f3-417f-b42f-f64a9ee91ea8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: 9d8630e6-683e-4fe9-acad-7602420ec371 kind: Event lastTimestamp: "2026-07-16T17:58:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T17:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:52Z" name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls.18c2d6f0d0a82399 namespace: kserve-ci-e2e-test resourceVersion: "21709" uid: 51dcd10c-d181-4a02-a10b-9b39a76ac4a9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: 9d8630e6-683e-4fe9-acad-7602420ec371 kind: Event lastTimestamp: "2026-07-16T17:58:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T17:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:52Z" name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls.18c2d6f0d3c7b46b namespace: kserve-ci-e2e-test resourceVersion: "21713" uid: a8b5d202-c937-4d51-9868-678db728a6f7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: 9d8630e6-683e-4fe9-acad-7602420ec371 kind: Event lastTimestamp: "2026-07-16T17:58:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T17:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T17:58:52Z" name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls.18c2d6f0d43faa1d namespace: kserve-ci-e2e-test resourceVersion: "21715" uid: ec27e7f5-8140-4304-97ee-e72b8fd339e3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-16T17:58:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: 9d8630e6-683e-4fe9-acad-7602420ec371 kind: Event lastTimestamp: "2026-07-16T18:00:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T17:58:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:00:03Z" name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls.18c2d6f1451b99dd namespace: kserve-ci-e2e-test resourceVersion: "22211" uid: c77e994c-2d9c-4459-ba92-7d9329af46e1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:00:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: 9d8630e6-683e-4fe9-acad-7602420ec371 kind: Event lastTimestamp: "2026-07-16T18:00:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T18:00:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:00:16Z" name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls.18c2d7048ef31d1b namespace: kserve-ci-e2e-test resourceVersion: "22332" uid: 60f90293-3745-4804-8d0b-e1d5a3448325 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:00:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: 9d8630e6-683e-4fe9-acad-7602420ec371 kind: Event lastTimestamp: "2026-07-16T18:00:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T18:00:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:00:16Z" name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls.18c2d7048ef39d2a namespace: kserve-ci-e2e-test resourceVersion: "22333" uid: 22d17471-b807-4380-8d2f-ac8dc6c3acea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:00:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: 9d8630e6-683e-4fe9-acad-7602420ec371 kind: Event lastTimestamp: "2026-07-16T18:00:20Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T18:00:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T18:00:20Z" name: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls.18c2d7054b53f442 namespace: kserve-ci-e2e-test resourceVersion: "22355" uid: 5ce7de36-f947-4749-93af-ba2c1a3fc0bd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-218.ec2.internal source: component: kubelet host: ip-10-0-132-218.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-9159d-predictor-bf65d4648 namespace: kserve-ci-e2e-test resourceVersion: "21627" uid: 4b54a0b2-b41c-4ec1-aeb5-e0747ced66d8 kind: Event lastTimestamp: "2026-07-16T17:58:46Z" message: 'Created pod: raw-sklearn-runtime-9159d-predictor-bf65d4648-cbjls' metadata: creationTimestamp: "2026-07-16T17:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:58:46Z" name: raw-sklearn-runtime-9159d-predictor-bf65d4648.18c2d6ef903ef5e3 namespace: kserve-ci-e2e-test resourceVersion: "21638" uid: 6b302a80-efbe-4b99-94c4-be785977056d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-9159d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21623" uid: 64a2d339-2d1c-423b-a974-44b81f1fff81 kind: Event lastTimestamp: "2026-07-16T17:58:46Z" message: Scaled up replica set raw-sklearn-runtime-9159d-predictor-bf65d4648 from 0 to 1 metadata: creationTimestamp: "2026-07-16T17:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:58:46Z" name: raw-sklearn-runtime-9159d-predictor.18c2d6ef8b8f7827 namespace: kserve-ci-e2e-test resourceVersion: "21629" uid: b42480f6-32c6-4035-ba54-8cdc461747b1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:59:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-9159d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21624" uid: 9f2e9c33-c36f-41c3-b8a8-b94e87b775cd kind: Event lastTimestamp: "2026-07-16T17:59:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:59:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:59:01Z" name: raw-sklearn-runtime-9159d-predictor.18c2d6f30b4e24a0 namespace: kserve-ci-e2e-test resourceVersion: "21782" uid: 52a6007b-51e1-4696-ab70-3cb702d7720e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:59:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-9159d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21624" uid: 9f2e9c33-c36f-41c3-b8a8-b94e87b775cd kind: Event lastTimestamp: "2026-07-16T17:59:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T17:59:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T17:59:01Z" name: raw-sklearn-runtime-9159d-predictor.18c2d6f30bdd2a5e namespace: kserve-ci-e2e-test resourceVersion: "21784" uid: 51cfe46d-4dfb-47d3-9ad6-65d01137131a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T17:59:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-9159d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21783" uid: 9f2e9c33-c36f-41c3-b8a8-b94e87b775cd kind: Event lastTimestamp: "2026-07-16T18:00:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:59:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:00:16Z" name: raw-sklearn-runtime-9159d-predictor.18c2d6f68aaadfdb namespace: kserve-ci-e2e-test resourceVersion: "22315" uid: 0cd4f622-710a-4bc3-a68c-7e378f7098a9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T17:59:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-9159d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21783" uid: 9f2e9c33-c36f-41c3-b8a8-b94e87b775cd kind: Event lastTimestamp: "2026-07-16T18:00:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T17:59:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T18:00:16Z" name: raw-sklearn-runtime-9159d-predictor.18c2d6f68b1f8fdd namespace: kserve-ci-e2e-test resourceVersion: "22316" uid: efb11495-9f96-4301-97bf-32558a82504a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-16T17:58:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-9159d namespace: kserve-ci-e2e-test resourceVersion: "21615" uid: 612ab9ed-8b7e-4ad3-bd7b-eddf09077d10 kind: Event lastTimestamp: "2026-07-16T18:00:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T17:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:00:13Z" name: raw-sklearn-runtime-9159d.18c2d6ef8b869b96 namespace: kserve-ci-e2e-test resourceVersion: "22293" uid: d3080861-c941-4e16-a1dd-29bb7e884d32 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T17:58:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-9159d namespace: kserve-ci-e2e-test resourceVersion: "21615" uid: 612ab9ed-8b7e-4ad3-bd7b-eddf09077d10 kind: Event lastTimestamp: "2026-07-16T17:58:46Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-9159d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-9159d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T17:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T17:58:46Z" name: raw-sklearn-runtime-9159d.18c2d6ef8c1f9320 namespace: kserve-ci-e2e-test resourceVersion: "21632" uid: 4f507516-d20c-41de-a6b9-aabe239813c3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T18:00:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-9159d namespace: kserve-ci-e2e-test resourceVersion: "22289" uid: 612ab9ed-8b7e-4ad3-bd7b-eddf09077d10 kind: Event lastTimestamp: "2026-07-16T18:00:13Z" message: InferenceService [raw-sklearn-runtime-9159d] is Ready metadata: creationTimestamp: "2026-07-16T18:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T18:00:13Z" name: raw-sklearn-runtime-9159d.18c2d703d0426c94 namespace: kserve-ci-e2e-test resourceVersion: "22290" uid: 0fc48923-f15b-4285-a6a0-bb2fcb9e0c88 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22581"