--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:19:17.399478Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-4dad45-predictor-866bdd48d5-h6p56 namespace: kserve-ci-e2e-test resourceVersion: "19273" uid: b9f10d81-6f9c-4742-9378-afa88662b0e3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-4dad45-predictor-866bdd48d5-h6p56 to ip-10-0-129-84.ec2.internal metadata: creationTimestamp: "2026-04-16T14:19:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:19:17Z" name: isvc-init-fail-4dad45-predictor-866bdd48d5-h6p56.18a6dc25abdc688b namespace: kserve-ci-e2e-test resourceVersion: "19277" uid: 8050d02e-041c-4d7d-87f8-cbcc10f4454a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-86d7fd6df4-tkh2q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:17Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-4dad45-predictor-866bdd48d5-h6p56 namespace: kserve-ci-e2e-test resourceVersion: "19281" uid: b9f10d81-6f9c-4742-9378-afa88662b0e3 kind: Event lastTimestamp: "2026-04-16T14:19:17Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:19:17Z" name: isvc-init-fail-4dad45-predictor-866bdd48d5-h6p56.18a6dc25c65e4dfb namespace: kserve-ci-e2e-test resourceVersion: "19297" uid: be0bfa0c-0dc7-40e4-a39c-48a7753d0252 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:19:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-4dad45-predictor-866bdd48d5-h6p56 namespace: kserve-ci-e2e-test resourceVersion: "19276" uid: b9f10d81-6f9c-4742-9378-afa88662b0e3 kind: Event lastTimestamp: "2026-04-16T14:19:24Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8eee296dba42148fd03e4b2602ce20b8f3cc18bf537b500919938742ad784fb1" already present on machine metadata: creationTimestamp: "2026-04-16T14:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:24Z" name: isvc-init-fail-4dad45-predictor-866bdd48d5-h6p56.18a6dc25c7a2e183 namespace: kserve-ci-e2e-test resourceVersion: "19369" uid: 77827304-d490-43ff-b7c3-ed9be131c021 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:19:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-4dad45-predictor-866bdd48d5-h6p56 namespace: kserve-ci-e2e-test resourceVersion: "19276" uid: b9f10d81-6f9c-4742-9378-afa88662b0e3 kind: Event lastTimestamp: "2026-04-16T14:19:24Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:24Z" name: isvc-init-fail-4dad45-predictor-866bdd48d5-h6p56.18a6dc25cab4163c namespace: kserve-ci-e2e-test resourceVersion: "19373" uid: b4ada106-3969-4a50-8f12-78e7ef0216f5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:19:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-4dad45-predictor-866bdd48d5-h6p56 namespace: kserve-ci-e2e-test resourceVersion: "19276" uid: b9f10d81-6f9c-4742-9378-afa88662b0e3 kind: Event lastTimestamp: "2026-04-16T14:19:24Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:24Z" name: isvc-init-fail-4dad45-predictor-866bdd48d5-h6p56.18a6dc25cb2958a3 namespace: kserve-ci-e2e-test resourceVersion: "19374" uid: 594ce868-de8d-444f-9470-99013007810f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-4dad45-predictor-866bdd48d5-h6p56 namespace: kserve-ci-e2e-test resourceVersion: "19276" uid: b9f10d81-6f9c-4742-9378-afa88662b0e3 kind: Event lastTimestamp: "2026-04-16T14:19:27Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-04-16T14:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:27Z" name: isvc-init-fail-4dad45-predictor-866bdd48d5-h6p56.18a6dc28001b930b namespace: kserve-ci-e2e-test resourceVersion: "19413" uid: a72d45b7-5477-4b84-9b75-9aa426e13cfa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-4dad45-predictor-866bdd48d5 namespace: kserve-ci-e2e-test resourceVersion: "19264" uid: d714df72-e623-406f-9fec-a18c8524e28c kind: Event lastTimestamp: "2026-04-16T14:19:17Z" message: 'Created pod: isvc-init-fail-4dad45-predictor-866bdd48d5-h6p56' metadata: creationTimestamp: "2026-04-16T14:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:19:17Z" name: isvc-init-fail-4dad45-predictor-866bdd48d5.18a6dc25ab705348 namespace: kserve-ci-e2e-test resourceVersion: "19274" uid: d54273b5-605e-4446-867a-97159387312c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-4dad45-predictor namespace: kserve-ci-e2e-test resourceVersion: "19262" uid: ca96d7be-9a41-4af7-81bc-66561e41d22b kind: Event lastTimestamp: "2026-04-16T14:19:17Z" message: Scaled up replica set isvc-init-fail-4dad45-predictor-866bdd48d5 from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:19:17Z" name: isvc-init-fail-4dad45-predictor.18a6dc25a7ce5aaa namespace: kserve-ci-e2e-test resourceVersion: "19266" uid: a521308a-fc3f-4528-8659-e9966c600ff4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T14:19:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-4dad45 namespace: kserve-ci-e2e-test resourceVersion: "19253" uid: f0278877-9ad6-4c6b-b91d-333dd35919ce kind: Event lastTimestamp: "2026-04-16T14:19:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:19:24Z" name: isvc-init-fail-4dad45.18a6dc25a7fd9da1 namespace: kserve-ci-e2e-test resourceVersion: "19375" uid: cb47758c-b8de-4b6b-b703-d7a4b98b3a83 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:19:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-4dad45 namespace: kserve-ci-e2e-test resourceVersion: "19253" uid: f0278877-9ad6-4c6b-b91d-333dd35919ce kind: Event lastTimestamp: "2026-04-16T14:19:17Z" message: 'Failed to update status for InferenceService "isvc-init-fail-4dad45": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-4dad45": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:19:17Z" name: isvc-init-fail-4dad45.18a6dc25a84fc0ed namespace: kserve-ci-e2e-test resourceVersion: "19288" uid: 087add24-147a-4ef1-bf10-a62f7f017579 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:19:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-4dad45 namespace: kserve-ci-e2e-test resourceVersion: "19253" uid: f0278877-9ad6-4c6b-b91d-333dd35919ce kind: Event lastTimestamp: "2026-04-16T14:19:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-4dad45": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:19:17Z" name: isvc-init-fail-4dad45.18a6dc25a8500640 namespace: kserve-ci-e2e-test resourceVersion: "19289" uid: 49fbc80c-261d-4c24-9160-5a1229401d78 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:13:55.470534Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg namespace: kserve-ci-e2e-test resourceVersion: "16607" uid: 1f1a4d5e-2dff-4de4-8da2-8062eebb190f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg to ip-10-0-129-84.ec2.internal metadata: creationTimestamp: "2026-04-16T14:13:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:13:55Z" name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg.18a6dbdab7668489 namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: fd2e775e-e315-412b-9a38-a0d44ac5d246 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-86d7fd6df4-tkh2q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:55Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg namespace: kserve-ci-e2e-test resourceVersion: "16616" uid: 1f1a4d5e-2dff-4de4-8da2-8062eebb190f kind: Event lastTimestamp: "2026-04-16T14:13:55Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:13:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:13:55Z" name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg.18a6dbdad0992f1d namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 91a40d47-9d6e-40d1-b06c-6bad7a471950 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: 1f1a4d5e-2dff-4de4-8da2-8062eebb190f kind: Event lastTimestamp: "2026-04-16T14:13:55Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8eee296dba42148fd03e4b2602ce20b8f3cc18bf537b500919938742ad784fb1" already present on machine metadata: creationTimestamp: "2026-04-16T14:13:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:55Z" name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg.18a6dbdad1eec333 namespace: kserve-ci-e2e-test resourceVersion: "16627" uid: 32f84a3f-ceb0-4d55-8d63-3284763a4ac8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: 1f1a4d5e-2dff-4de4-8da2-8062eebb190f kind: Event lastTimestamp: "2026-04-16T14:13:55Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:13:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:55Z" name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg.18a6dbdad57ef09a namespace: kserve-ci-e2e-test resourceVersion: "16629" uid: 90d08df7-7b32-4039-b89d-01b7e647340e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: 1f1a4d5e-2dff-4de4-8da2-8062eebb190f kind: Event lastTimestamp: "2026-04-16T14:13:55Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:13:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:55Z" name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg.18a6dbdad5ed2f9f namespace: kserve-ci-e2e-test resourceVersion: "16630" uid: c3f80892-90e4-4775-a65d-eb988c2eb05d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:14:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: 1f1a4d5e-2dff-4de4-8da2-8062eebb190f kind: Event lastTimestamp: "2026-04-16T14:14:00Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1424" already present on machine metadata: creationTimestamp: "2026-04-16T14:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:14:00Z" name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg.18a6dbdbecefb40e namespace: kserve-ci-e2e-test resourceVersion: "16676" uid: 52273466-9100-4fa5-bb68-84f82e452692 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:14:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: 1f1a4d5e-2dff-4de4-8da2-8062eebb190f kind: Event lastTimestamp: "2026-04-16T14:14:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:14:00Z" name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg.18a6dbdbf037b913 namespace: kserve-ci-e2e-test resourceVersion: "16680" uid: 5c856ad6-e4d1-4cd6-aa2c-5144b8d796e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:14:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: 1f1a4d5e-2dff-4de4-8da2-8062eebb190f kind: Event lastTimestamp: "2026-04-16T14:14:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:14:00Z" name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg.18a6dbdbf0acd033 namespace: kserve-ci-e2e-test resourceVersion: "16681" uid: 0aa87700-1168-456a-86d7-b049a471d3b2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:14:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: 1f1a4d5e-2dff-4de4-8da2-8062eebb190f kind: Event lastTimestamp: "2026-04-16T14:14:00Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:047f63b6a928a84a5cb6317b9490b65e3210ada3d182f71aa6a7a838eb4e1ff7" already present on machine metadata: creationTimestamp: "2026-04-16T14:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:14:00Z" name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg.18a6dbdbf0b91d92 namespace: kserve-ci-e2e-test resourceVersion: "16682" uid: 4e18ea63-5849-468a-9ffa-d10da4a1cb9c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:14:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: 1f1a4d5e-2dff-4de4-8da2-8062eebb190f kind: Event lastTimestamp: "2026-04-16T14:14:00Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-16T14:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:14:00Z" name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg.18a6dbdbf3e03e54 namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: 88d39eab-5d73-4105-84d2-4bbfdbdc5db3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:14:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: 1f1a4d5e-2dff-4de4-8da2-8062eebb190f kind: Event lastTimestamp: "2026-04-16T14:14:00Z" message: Started container agent metadata: creationTimestamp: "2026-04-16T14:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:14:00Z" name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg.18a6dbdbf4517cc9 namespace: kserve-ci-e2e-test resourceVersion: "16686" uid: 7fbd4fe3-4cc9-4012-b4d8-b97677759306 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-16T14:14:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: 1f1a4d5e-2dff-4de4-8da2-8062eebb190f kind: Event lastTimestamp: "2026-04-16T14:15:33Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T14:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:15:33Z" name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg.18a6dbdc28e36d05 namespace: kserve-ci-e2e-test resourceVersion: "17472" uid: 717abe8e-e2ff-45fb-8d07-39c54e83ef16 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T14:14:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: 1f1a4d5e-2dff-4de4-8da2-8062eebb190f kind: Event lastTimestamp: "2026-04-16T14:15:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T14:14:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:15:33Z" name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg.18a6dbdc64a7e0be namespace: kserve-ci-e2e-test resourceVersion: "17473" uid: 5e681620-0e68-4d64-8048-81bcad7eca63 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:15:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: 1f1a4d5e-2dff-4de4-8da2-8062eebb190f kind: Event lastTimestamp: "2026-04-16T14:15:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:15:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:15:10Z" name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg.18a6dbec3d569978 namespace: kserve-ci-e2e-test resourceVersion: "17249" uid: 7c3baef0-2a14-4f49-b2a3-f96386489b57 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:15:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: 1f1a4d5e-2dff-4de4-8da2-8062eebb190f kind: Event lastTimestamp: "2026-04-16T14:15:10Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-16T14:15:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:15:10Z" name: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg.18a6dbec3d57ede9 namespace: kserve-ci-e2e-test resourceVersion: "17251" uid: ca1fdf1d-2d7c-4d51-b310-016d56b7593f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-a0d01-predictor-77cdd7f44d namespace: kserve-ci-e2e-test resourceVersion: "16595" uid: 9c782edc-3b19-4410-9b71-4215ef69f7ad kind: Event lastTimestamp: "2026-04-16T14:13:55Z" message: 'Created pod: isvc-logger-raw-a0d01-predictor-77cdd7f44d-x49kg' metadata: creationTimestamp: "2026-04-16T14:13:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:55Z" name: isvc-logger-raw-a0d01-predictor-77cdd7f44d.18a6dbdab6fb21e9 namespace: kserve-ci-e2e-test resourceVersion: "16608" uid: 7505cd51-8736-4808-a228-eae3a5c30a64 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-a0d01-predictor namespace: kserve-ci-e2e-test resourceVersion: "16593" uid: 0e74f9f8-7540-4f29-af98-803512e41a70 kind: Event lastTimestamp: "2026-04-16T14:13:55Z" message: Scaled up replica set isvc-logger-raw-a0d01-predictor-77cdd7f44d from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:13:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:55Z" name: isvc-logger-raw-a0d01-predictor.18a6dbdab4200cb1 namespace: kserve-ci-e2e-test resourceVersion: "16597" uid: b94cedc1-7fab-48dc-a729-160c7bc8352f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:14:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-a0d01-predictor namespace: kserve-ci-e2e-test resourceVersion: "16596" uid: 34536c62-4a01-4ea3-a9a7-04f661a02c25 kind: Event lastTimestamp: "2026-04-16T14:14:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:14:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:14:10Z" name: isvc-logger-raw-a0d01-predictor.18a6dbde32c35fdc namespace: kserve-ci-e2e-test resourceVersion: "16765" uid: 22a494f8-c6ba-4364-9993-b163fd36b752 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:14:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-a0d01-predictor namespace: kserve-ci-e2e-test resourceVersion: "16596" uid: 34536c62-4a01-4ea3-a9a7-04f661a02c25 kind: Event lastTimestamp: "2026-04-16T14:14:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:14:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:14:10Z" name: isvc-logger-raw-a0d01-predictor.18a6dbde3320d697 namespace: kserve-ci-e2e-test resourceVersion: "16767" uid: 1d03c3f8-16da-4ddd-942c-6c8dc05c4ed7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T14:14:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-a0d01-predictor namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: 34536c62-4a01-4ea3-a9a7-04f661a02c25 kind: Event lastTimestamp: "2026-04-16T14:15:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:14:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:15:10Z" name: isvc-logger-raw-a0d01-predictor.18a6dbe1b1c31d92 namespace: kserve-ci-e2e-test resourceVersion: "17208" uid: e0f450e8-4e74-4a04-ad45-387d5f96ac75 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T14:14:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-a0d01-predictor namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: 34536c62-4a01-4ea3-a9a7-04f661a02c25 kind: Event lastTimestamp: "2026-04-16T14:15:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:14:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:15:10Z" name: isvc-logger-raw-a0d01-predictor.18a6dbe1b2262e18 namespace: kserve-ci-e2e-test resourceVersion: "17210" uid: 5a09a70f-790f-4231-932c-2937f6b4c8ef reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-04-16T14:13:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-a0d01 namespace: kserve-ci-e2e-test resourceVersion: "16586" uid: 35884bbf-30f1-4dd8-8c5f-a56342fb540a kind: Event lastTimestamp: "2026-04-16T14:15:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:13:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:15:03Z" name: isvc-logger-raw-a0d01.18a6dbdab4349def namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: 06dd7a44-a235-41ac-acf4-2460b8aacd87 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:13:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-a0d01 namespace: kserve-ci-e2e-test resourceVersion: "16586" uid: 35884bbf-30f1-4dd8-8c5f-a56342fb540a kind: Event lastTimestamp: "2026-04-16T14:13:55Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-a0d01": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-a0d01": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:13:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:13:55Z" name: isvc-logger-raw-a0d01.18a6dbdab4b008ee namespace: kserve-ci-e2e-test resourceVersion: "16617" uid: 3391f8b8-deb1-4c52-bff5-e78fd77a76cc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-a0d01 namespace: kserve-ci-e2e-test resourceVersion: "16594" uid: 35884bbf-30f1-4dd8-8c5f-a56342fb540a kind: Event lastTimestamp: "2026-04-16T14:13:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-a0d01": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:13:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:13:55Z" name: isvc-logger-raw-a0d01.18a6dbdab89cd230 namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: 2d1a129e-27c8-429e-9df2-7784edce0be7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:15:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-a0d01 namespace: kserve-ci-e2e-test resourceVersion: "17164" uid: 35884bbf-30f1-4dd8-8c5f-a56342fb540a kind: Event lastTimestamp: "2026-04-16T14:15:03Z" message: InferenceService [isvc-logger-raw-a0d01] is Ready metadata: creationTimestamp: "2026-04-16T14:15:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:15:03Z" name: isvc-logger-raw-a0d01.18a6dbea9d995220 namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: 5ead8c2a-edd8-49fb-ac89-800613bf622d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:17:41.056314Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf namespace: kserve-ci-e2e-test resourceVersion: "18419" uid: bbade444-703c-48fb-9168-bcbfc7deaea4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf to ip-10-0-129-84.ec2.internal metadata: creationTimestamp: "2026-04-16T14:17:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:17:41Z" name: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf.18a6dc0f3d5c5980 namespace: kserve-ci-e2e-test resourceVersion: "18426" uid: 2cc25a29-7791-46f3-b2f4-6ad279fdf240 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-86d7fd6df4-tkh2q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:41Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf namespace: kserve-ci-e2e-test resourceVersion: "18429" uid: bbade444-703c-48fb-9168-bcbfc7deaea4 kind: Event lastTimestamp: "2026-04-16T14:17:41Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:17:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:17:41Z" name: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf.18a6dc0f56ea7fcc namespace: kserve-ci-e2e-test resourceVersion: "18439" uid: b7b922a9-7651-4753-b139-17682233b8c0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf namespace: kserve-ci-e2e-test resourceVersion: "18421" uid: bbade444-703c-48fb-9168-bcbfc7deaea4 kind: Event lastTimestamp: "2026-04-16T14:17:41Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8eee296dba42148fd03e4b2602ce20b8f3cc18bf537b500919938742ad784fb1" already present on machine metadata: creationTimestamp: "2026-04-16T14:17:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:17:41Z" name: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf.18a6dc0f5830aa26 namespace: kserve-ci-e2e-test resourceVersion: "18441" uid: a7f437dd-1fb3-4d6a-816f-3f20874adea7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf namespace: kserve-ci-e2e-test resourceVersion: "18421" uid: bbade444-703c-48fb-9168-bcbfc7deaea4 kind: Event lastTimestamp: "2026-04-16T14:17:41Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:17:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:17:41Z" name: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf.18a6dc0f5bf385d7 namespace: kserve-ci-e2e-test resourceVersion: "18443" uid: 975ca619-cdd8-4aa4-aafa-79ccdcd822f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf namespace: kserve-ci-e2e-test resourceVersion: "18421" uid: bbade444-703c-48fb-9168-bcbfc7deaea4 kind: Event lastTimestamp: "2026-04-16T14:17:41Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:17:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:17:41Z" name: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf.18a6dc0f5c693fb3 namespace: kserve-ci-e2e-test resourceVersion: "18445" uid: 10b0b34f-a172-462b-b1be-98f7a45cd40d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf namespace: kserve-ci-e2e-test resourceVersion: "18421" uid: bbade444-703c-48fb-9168-bcbfc7deaea4 kind: Event lastTimestamp: "2026-04-16T14:17:45Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1424" already present on machine metadata: creationTimestamp: "2026-04-16T14:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:17:45Z" name: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf.18a6dc10434358f0 namespace: kserve-ci-e2e-test resourceVersion: "18478" uid: bf6cdcd3-f5cd-421f-adf3-c29f5f92a2a7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf namespace: kserve-ci-e2e-test resourceVersion: "18421" uid: bbade444-703c-48fb-9168-bcbfc7deaea4 kind: Event lastTimestamp: "2026-04-16T14:17:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:17:45Z" name: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf.18a6dc1046ba8875 namespace: kserve-ci-e2e-test resourceVersion: "18482" uid: a88159b7-72d9-4139-93c4-b52a0272a106 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf namespace: kserve-ci-e2e-test resourceVersion: "18421" uid: bbade444-703c-48fb-9168-bcbfc7deaea4 kind: Event lastTimestamp: "2026-04-16T14:17:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:17:45Z" name: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf.18a6dc10472b8a4c namespace: kserve-ci-e2e-test resourceVersion: "18484" uid: 92a7671e-f44c-4ac0-833e-28a7ca3a8251 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T14:17:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf namespace: kserve-ci-e2e-test resourceVersion: "18421" uid: bbade444-703c-48fb-9168-bcbfc7deaea4 kind: Event lastTimestamp: "2026-04-16T14:18:47Z" message: 'Readiness probe failed: dial tcp 10.132.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T14:17:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:18:47Z" name: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf.18a6dc107f2d90a5 namespace: kserve-ci-e2e-test resourceVersion: "18922" uid: 226fae79-e773-4fc7-916a-c4dabfc9a389 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf namespace: kserve-ci-e2e-test resourceVersion: "18421" uid: bbade444-703c-48fb-9168-bcbfc7deaea4 kind: Event lastTimestamp: "2026-04-16T14:19:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:17Z" name: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf.18a6dc25a1318430 namespace: kserve-ci-e2e-test resourceVersion: "19236" uid: ddce2872-f94b-4877-8870-d71ba7c10651 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-8047bd-predictor-6f97f8bc7d namespace: kserve-ci-e2e-test resourceVersion: "18409" uid: 34046ce2-c731-4099-b0bc-d640dd722f73 kind: Event lastTimestamp: "2026-04-16T14:17:41Z" message: 'Created pod: isvc-primary-8047bd-predictor-6f97f8bc7d-4clgf' metadata: creationTimestamp: "2026-04-16T14:17:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:17:41Z" name: isvc-primary-8047bd-predictor-6f97f8bc7d.18a6dc0f3cc1ff5a namespace: kserve-ci-e2e-test resourceVersion: "18420" uid: 0e8e6808-0ac6-4e6c-8056-da11ee285184 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-8047bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "18406" uid: 2b7bc840-c15c-4a65-a01a-edf15a772e67 kind: Event lastTimestamp: "2026-04-16T14:17:40Z" message: Scaled up replica set isvc-primary-8047bd-predictor-6f97f8bc7d from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:17:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:17:40Z" name: isvc-primary-8047bd-predictor.18a6dc0f3859f9c4 namespace: kserve-ci-e2e-test resourceVersion: "18412" uid: 97514249-f16c-4b64-963a-9dd2a39edb2b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:17:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-8047bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "18408" uid: c1e09275-f86e-47e3-b71f-6776cf55a5e9 kind: Event lastTimestamp: "2026-04-16T14:18:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:18:26Z" name: isvc-primary-8047bd-predictor.18a6dc12b71df261 namespace: kserve-ci-e2e-test resourceVersion: "18766" uid: d796434d-b14d-4f35-9859-7ae3d468db1b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:17:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-8047bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "18408" uid: c1e09275-f86e-47e3-b71f-6776cf55a5e9 kind: Event lastTimestamp: "2026-04-16T14:18:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:18:26Z" name: isvc-primary-8047bd-predictor.18a6dc12b787eb96 namespace: kserve-ci-e2e-test resourceVersion: "18767" uid: daaadba0-840c-49c9-87fe-1bcf4fd759cb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:18:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-8047bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "18560" uid: c1e09275-f86e-47e3-b71f-6776cf55a5e9 kind: Event lastTimestamp: "2026-04-16T14:19:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:18:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:19:11Z" name: isvc-primary-8047bd-predictor.18a6dc1d3487a8d7 namespace: kserve-ci-e2e-test resourceVersion: "19167" uid: dda13c89-5e51-4d8c-a8d6-6352b8cee11e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:18:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-8047bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "18560" uid: c1e09275-f86e-47e3-b71f-6776cf55a5e9 kind: Event lastTimestamp: "2026-04-16T14:19:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:18:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:19:11Z" name: isvc-primary-8047bd-predictor.18a6dc1d34e7e070 namespace: kserve-ci-e2e-test resourceVersion: "19168" uid: 2f5e5633-66f6-4319-a4db-7771b465ec96 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T14:17:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-8047bd namespace: kserve-ci-e2e-test resourceVersion: "18382" uid: e5a1a87f-d7c2-41da-b081-5dde8f3373cd kind: Event lastTimestamp: "2026-04-16T14:18:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:17:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:18:56Z" name: isvc-primary-8047bd.18a6dc0f3834100c namespace: kserve-ci-e2e-test resourceVersion: "18999" uid: 49ddc803-6613-4a04-84b8-d6a8866c682b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:17:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-8047bd namespace: kserve-ci-e2e-test resourceVersion: "18382" uid: e5a1a87f-d7c2-41da-b081-5dde8f3373cd kind: Event lastTimestamp: "2026-04-16T14:17:41Z" message: 'Failed to update status for InferenceService "isvc-primary-8047bd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-8047bd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:17:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:17:41Z" name: isvc-primary-8047bd.18a6dc0f38c96a9c namespace: kserve-ci-e2e-test resourceVersion: "18425" uid: 69330f00-78d3-4fea-9135-f0de88cbf731 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:17:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-8047bd namespace: kserve-ci-e2e-test resourceVersion: "18382" uid: e5a1a87f-d7c2-41da-b081-5dde8f3373cd kind: Event lastTimestamp: "2026-04-16T14:17:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-8047bd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:17:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:17:41Z" name: isvc-primary-8047bd.18a6dc0f38c9c75b namespace: kserve-ci-e2e-test resourceVersion: "18427" uid: 3c609acc-a47f-4862-b485-c1ea2557006c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:18:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-8047bd namespace: kserve-ci-e2e-test resourceVersion: "18997" uid: e5a1a87f-d7c2-41da-b081-5dde8f3373cd kind: Event lastTimestamp: "2026-04-16T14:18:56Z" message: InferenceService [isvc-primary-8047bd] is Ready metadata: creationTimestamp: "2026-04-16T14:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:18:56Z" name: isvc-primary-8047bd.18a6dc20d7cccd93 namespace: kserve-ci-e2e-test resourceVersion: "18998" uid: f3b951ee-3ce8-4f27-bf31-41664a08cb84 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:08:49.674075Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs namespace: kserve-ci-e2e-test resourceVersion: "13790" uid: 5ef18018-5b9a-4799-919f-27e187f05a2b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs to ip-10-0-129-84.ec2.internal metadata: creationTimestamp: "2026-04-16T14:08:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:08:49Z" name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs.18a6db938482e597 namespace: kserve-ci-e2e-test resourceVersion: "13796" uid: 10c796a3-9a17-49f2-b365-35e7adb3a69e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-86d7fd6df4-tkh2q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs namespace: kserve-ci-e2e-test resourceVersion: "13802" uid: 5ef18018-5b9a-4799-919f-27e187f05a2b kind: Event lastTimestamp: "2026-04-16T14:08:50Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:08:50Z" name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs.18a6db939e50c2ba namespace: kserve-ci-e2e-test resourceVersion: "13813" uid: 11360332-95a8-402f-bb8d-fb9e4272eff4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: 5ef18018-5b9a-4799-919f-27e187f05a2b kind: Event lastTimestamp: "2026-04-16T14:08:50Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:8eee296dba42148fd03e4b2602ce20b8f3cc18bf537b500919938742ad784fb1" metadata: creationTimestamp: "2026-04-16T14:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:08:50Z" name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs.18a6db939f8376e9 namespace: kserve-ci-e2e-test resourceVersion: "13815" uid: 5e87edd5-9124-4a7c-a225-6791da2d2e61 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: 5ef18018-5b9a-4799-919f-27e187f05a2b kind: Event lastTimestamp: "2026-04-16T14:08:53Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:8eee296dba42148fd03e4b2602ce20b8f3cc18bf537b500919938742ad784fb1" in 3.151s (3.151s including waiting). Image size: 299668405 bytes.' metadata: creationTimestamp: "2026-04-16T14:08:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:08:53Z" name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs.18a6db945b606962 namespace: kserve-ci-e2e-test resourceVersion: "13835" uid: 54ca793b-d313-4436-af38-3c5a4d2c3890 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: 5ef18018-5b9a-4799-919f-27e187f05a2b kind: Event lastTimestamp: "2026-04-16T14:08:53Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:08:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:08:53Z" name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs.18a6db945eb9c396 namespace: kserve-ci-e2e-test resourceVersion: "13837" uid: 6d7c229a-b416-4f7c-95eb-a88fe3fc3649 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: 5ef18018-5b9a-4799-919f-27e187f05a2b kind: Event lastTimestamp: "2026-04-16T14:08:53Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:08:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:08:53Z" name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs.18a6db945f297ddb namespace: kserve-ci-e2e-test resourceVersion: "13838" uid: 2fb39327-5531-4f4d-9263-33907c6d7051 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: 5ef18018-5b9a-4799-919f-27e187f05a2b kind: Event lastTimestamp: "2026-04-16T14:08:57Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1424" metadata: creationTimestamp: "2026-04-16T14:08:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:08:57Z" name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs.18a6db955a487588 namespace: kserve-ci-e2e-test resourceVersion: "13878" uid: 8024fb48-8201-4a43-834f-79b58bb97f9d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: 5ef18018-5b9a-4799-919f-27e187f05a2b kind: Event lastTimestamp: "2026-04-16T14:09:11Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1424" in 13.593s (13.593s including waiting). Image size: 1560613808 bytes.' metadata: creationTimestamp: "2026-04-16T14:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:09:11Z" name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs.18a6db9884837448 namespace: kserve-ci-e2e-test resourceVersion: "13974" uid: 66542406-442f-40f4-ab51-67c8f34dbf0c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: 5ef18018-5b9a-4799-919f-27e187f05a2b kind: Event lastTimestamp: "2026-04-16T14:09:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:09:11Z" name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs.18a6db988820ddd2 namespace: kserve-ci-e2e-test resourceVersion: "13975" uid: 2c9f764e-3270-456e-a899-ffca114461a5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: 5ef18018-5b9a-4799-919f-27e187f05a2b kind: Event lastTimestamp: "2026-04-16T14:09:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:09:11Z" name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs.18a6db98889279e3 namespace: kserve-ci-e2e-test resourceVersion: "13976" uid: e82c0c17-cbfc-4784-b251-6d6a12f6937b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: 5ef18018-5b9a-4799-919f-27e187f05a2b kind: Event lastTimestamp: "2026-04-16T14:09:11Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:047f63b6a928a84a5cb6317b9490b65e3210ada3d182f71aa6a7a838eb4e1ff7" metadata: creationTimestamp: "2026-04-16T14:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:09:11Z" name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs.18a6db98889ebe22 namespace: kserve-ci-e2e-test resourceVersion: "13977" uid: 7fc7ba5a-bb53-475b-b6ee-145f3c1f1083 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: 5ef18018-5b9a-4799-919f-27e187f05a2b kind: Event lastTimestamp: "2026-04-16T14:09:14Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:047f63b6a928a84a5cb6317b9490b65e3210ada3d182f71aa6a7a838eb4e1ff7" in 3.262s (3.262s including waiting). Image size: 237896279 bytes.' metadata: creationTimestamp: "2026-04-16T14:09:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:09:14Z" name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs.18a6db994b0de274 namespace: kserve-ci-e2e-test resourceVersion: "14002" uid: 1a8c0240-9c82-4e14-940d-5a14f9e61ac9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: 5ef18018-5b9a-4799-919f-27e187f05a2b kind: Event lastTimestamp: "2026-04-16T14:09:14Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-16T14:09:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:09:14Z" name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs.18a6db994f3fefc0 namespace: kserve-ci-e2e-test resourceVersion: "14003" uid: d6509d78-6037-4d40-869e-204c70e0f0f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:09:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: 5ef18018-5b9a-4799-919f-27e187f05a2b kind: Event lastTimestamp: "2026-04-16T14:09:14Z" message: Started container agent metadata: creationTimestamp: "2026-04-16T14:09:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:09:14Z" name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs.18a6db994fb6892f namespace: kserve-ci-e2e-test resourceVersion: "14004" uid: d7d19443-313c-4182-b5a2-a64f0082c962 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T14:09:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: 5ef18018-5b9a-4799-919f-27e187f05a2b kind: Event lastTimestamp: "2026-04-16T14:10:45Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T14:09:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:45Z" name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs.18a6db99540c326d namespace: kserve-ci-e2e-test resourceVersion: "14801" uid: f7533959-7ebb-4ab4-9899-4de2c014017a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T14:09:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: 5ef18018-5b9a-4799-919f-27e187f05a2b kind: Event lastTimestamp: "2026-04-16T14:10:45Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T14:09:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:45Z" name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs.18a6db9954160943 namespace: kserve-ci-e2e-test resourceVersion: "14802" uid: 539af4b8-2e17-470e-b0d7-20a0ffa2fd35 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: 5ef18018-5b9a-4799-919f-27e187f05a2b kind: Event lastTimestamp: "2026-04-16T14:10:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:24Z" name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs.18a6dba9a9ebfd7b namespace: kserve-ci-e2e-test resourceVersion: "14550" uid: bf44de40-062f-4b41-a092-d317547f570a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: 5ef18018-5b9a-4799-919f-27e187f05a2b kind: Event lastTimestamp: "2026-04-16T14:10:24Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-16T14:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:24Z" name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs.18a6dba9a9ed6dbf namespace: kserve-ci-e2e-test resourceVersion: "14552" uid: 03b07dea-c5ab-4275-ab38-5abfec3c31be reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c namespace: kserve-ci-e2e-test resourceVersion: "13782" uid: 59adfb50-5b46-4460-8bfe-9d87da4c1a90 kind: Event lastTimestamp: "2026-04-16T14:08:49Z" message: 'Created pod: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c-7jzfs' metadata: creationTimestamp: "2026-04-16T14:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:08:49Z" name: isvc-raw-sklearn-batcher-72692-predictor-566c5d979c.18a6db9384086cd1 namespace: kserve-ci-e2e-test resourceVersion: "13792" uid: e1f9e0e1-5d45-44ac-a7ca-373debcf570d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-72692-predictor namespace: kserve-ci-e2e-test resourceVersion: "13781" uid: 5b001505-acf2-435a-ac38-2674eb7c5632 kind: Event lastTimestamp: "2026-04-16T14:08:49Z" message: Scaled up replica set isvc-raw-sklearn-batcher-72692-predictor-566c5d979c from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:08:49Z" name: isvc-raw-sklearn-batcher-72692-predictor.18a6db937e450efc namespace: kserve-ci-e2e-test resourceVersion: "13783" uid: ed27b64e-4dce-47fd-a3dd-80af1ea42802 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T14:09:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-72692-predictor namespace: kserve-ci-e2e-test resourceVersion: "13791" uid: aeb569ee-bb44-4a61-a850-0bc42a46656c kind: Event lastTimestamp: "2026-04-16T14:10:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:09:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:10:04Z" name: isvc-raw-sklearn-batcher-72692-predictor.18a6db9703895615 namespace: kserve-ci-e2e-test resourceVersion: "14361" uid: 2c7cf797-9777-4ca2-9f81-1ad5ece1c0c9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T14:09:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-72692-predictor namespace: kserve-ci-e2e-test resourceVersion: "13791" uid: aeb569ee-bb44-4a61-a850-0bc42a46656c kind: Event lastTimestamp: "2026-04-16T14:10:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:09:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:10:04Z" name: isvc-raw-sklearn-batcher-72692-predictor.18a6db9703d531ea namespace: kserve-ci-e2e-test resourceVersion: "14362" uid: 453bee28-2c7f-4138-8e6b-763119e68ad4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-72692-predictor namespace: kserve-ci-e2e-test resourceVersion: "13932" uid: aeb569ee-bb44-4a61-a850-0bc42a46656c kind: Event lastTimestamp: "2026-04-16T14:10:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:10:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:10:19Z" name: isvc-raw-sklearn-batcher-72692-predictor.18a6dba87dfd226d namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: 3c9b675b-bec0-4f84-98e0-2589ec12dcaa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-72692-predictor namespace: kserve-ci-e2e-test resourceVersion: "13932" uid: aeb569ee-bb44-4a61-a850-0bc42a46656c kind: Event lastTimestamp: "2026-04-16T14:10:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:10:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:10:19Z" name: isvc-raw-sklearn-batcher-72692-predictor.18a6dba87e5e56b9 namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: e8652022-cad4-4208-be95-7342183bddbf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T14:08:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-72692 namespace: kserve-ci-e2e-test resourceVersion: "13770" uid: e229e124-6fc1-466a-a7c7-771653e39fec kind: Event lastTimestamp: "2026-04-16T14:10:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:10:15Z" name: isvc-raw-sklearn-batcher-72692.18a6db938458ccc2 namespace: kserve-ci-e2e-test resourceVersion: "14450" uid: 4a2f1aa9-6659-4dfb-a2df-3d93b4513418 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-72692 namespace: kserve-ci-e2e-test resourceVersion: "13770" uid: e229e124-6fc1-466a-a7c7-771653e39fec kind: Event lastTimestamp: "2026-04-16T14:08:49Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-72692": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-72692": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:08:49Z" name: isvc-raw-sklearn-batcher-72692.18a6db9384ad2926 namespace: kserve-ci-e2e-test resourceVersion: "13797" uid: 1fe2b96f-271e-4b37-ae38-7aa25b7eaa69 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:08:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-72692 namespace: kserve-ci-e2e-test resourceVersion: "13770" uid: e229e124-6fc1-466a-a7c7-771653e39fec kind: Event lastTimestamp: "2026-04-16T14:08:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-72692": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:08:49Z" name: isvc-raw-sklearn-batcher-72692.18a6db9384ad7888 namespace: kserve-ci-e2e-test resourceVersion: "13800" uid: 6a1dbf3f-a169-401f-8ec7-ae83c4abe40a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-72692 namespace: kserve-ci-e2e-test resourceVersion: "14448" uid: e229e124-6fc1-466a-a7c7-771653e39fec kind: Event lastTimestamp: "2026-04-16T14:10:15Z" message: InferenceService [isvc-raw-sklearn-batcher-72692] is Ready metadata: creationTimestamp: "2026-04-16T14:10:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:10:15Z" name: isvc-raw-sklearn-batcher-72692.18a6dba78cd8847e namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 2e3ace6b-b437-41e6-9751-7ea79bf08e79 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:19:01.180802Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-8047bd-predictor-7b7bdccd9f-cnrp9 namespace: kserve-ci-e2e-test resourceVersion: "19062" uid: 8685b553-2c3e-4882-a5f8-af6b2c69fafb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-8047bd-predictor-7b7bdccd9f-cnrp9 to ip-10-0-129-84.ec2.internal metadata: creationTimestamp: "2026-04-16T14:19:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:19:01Z" name: isvc-secondary-8047bd-predictor-7b7bdccd9f-cnrp9.18a6dc21e526f6b1 namespace: kserve-ci-e2e-test resourceVersion: "19067" uid: d556d5dd-5292-4f70-ae03-a7a5f54f1fa4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-86d7fd6df4-tkh2q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-8047bd-predictor-7b7bdccd9f-cnrp9 namespace: kserve-ci-e2e-test resourceVersion: "19072" uid: 8685b553-2c3e-4882-a5f8-af6b2c69fafb kind: Event lastTimestamp: "2026-04-16T14:19:01Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:19:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:19:01Z" name: isvc-secondary-8047bd-predictor-7b7bdccd9f-cnrp9.18a6dc21fe2b18f6 namespace: kserve-ci-e2e-test resourceVersion: "19081" uid: 260544ab-d460-47bf-b77d-1a68aabb6861 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:19:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-8047bd-predictor-7b7bdccd9f-cnrp9 namespace: kserve-ci-e2e-test resourceVersion: "19065" uid: 8685b553-2c3e-4882-a5f8-af6b2c69fafb kind: Event lastTimestamp: "2026-04-16T14:19:05Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8eee296dba42148fd03e4b2602ce20b8f3cc18bf537b500919938742ad784fb1" already present on machine metadata: creationTimestamp: "2026-04-16T14:19:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:05Z" name: isvc-secondary-8047bd-predictor-7b7bdccd9f-cnrp9.18a6dc21ff62f809 namespace: kserve-ci-e2e-test resourceVersion: "19118" uid: 03967c29-8305-40db-a9f7-1ec4a0a7eb05 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:19:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-8047bd-predictor-7b7bdccd9f-cnrp9 namespace: kserve-ci-e2e-test resourceVersion: "19065" uid: 8685b553-2c3e-4882-a5f8-af6b2c69fafb kind: Event lastTimestamp: "2026-04-16T14:19:05Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:19:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:05Z" name: isvc-secondary-8047bd-predictor-7b7bdccd9f-cnrp9.18a6dc220257440d namespace: kserve-ci-e2e-test resourceVersion: "19122" uid: 877d9d2d-22d0-41c8-930b-88b8342e3118 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:19:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-8047bd-predictor-7b7bdccd9f-cnrp9 namespace: kserve-ci-e2e-test resourceVersion: "19065" uid: 8685b553-2c3e-4882-a5f8-af6b2c69fafb kind: Event lastTimestamp: "2026-04-16T14:19:05Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:19:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:05Z" name: isvc-secondary-8047bd-predictor-7b7bdccd9f-cnrp9.18a6dc2202c81320 namespace: kserve-ci-e2e-test resourceVersion: "19123" uid: 2b4fda11-b634-4553-863d-efbac5f7eb02 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-8047bd-predictor-7b7bdccd9f-cnrp9 namespace: kserve-ci-e2e-test resourceVersion: "19065" uid: 8685b553-2c3e-4882-a5f8-af6b2c69fafb kind: Event lastTimestamp: "2026-04-16T14:19:07Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-8047bd-predictor-7b7bdccd9f-cnrp9_kserve-ci-e2e-test(8685b553-2c3e-4882-a5f8-af6b2c69fafb) metadata: creationTimestamp: "2026-04-16T14:19:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:07Z" name: isvc-secondary-8047bd-predictor-7b7bdccd9f-cnrp9.18a6dc236d0a0add namespace: kserve-ci-e2e-test resourceVersion: "19139" uid: b16f9e82-da96-4c9b-b7e1-34e5046fa6a8 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-8047bd-predictor-7b7bdccd9f namespace: kserve-ci-e2e-test resourceVersion: "19051" uid: f46af807-12ae-410b-869e-fbf2528e6872 kind: Event lastTimestamp: "2026-04-16T14:19:01Z" message: 'Created pod: isvc-secondary-8047bd-predictor-7b7bdccd9f-cnrp9' metadata: creationTimestamp: "2026-04-16T14:19:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:19:01Z" name: isvc-secondary-8047bd-predictor-7b7bdccd9f.18a6dc21e490f47b namespace: kserve-ci-e2e-test resourceVersion: "19063" uid: 00a34a39-afb5-4526-b4b8-3cdd954fc284 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-8047bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "19049" uid: d37e8347-e0e1-4285-a69f-aa392d2c2e42 kind: Event lastTimestamp: "2026-04-16T14:19:01Z" message: Scaled up replica set isvc-secondary-8047bd-predictor-7b7bdccd9f from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:19:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:19:01Z" name: isvc-secondary-8047bd-predictor.18a6dc21e1c889a1 namespace: kserve-ci-e2e-test resourceVersion: "19054" uid: 3c413752-d791-40d9-a30c-07361a654f1b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-8047bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "19050" uid: af33a606-4961-4b05-8b99-0db27897fff7 kind: Event lastTimestamp: "2026-04-16T14:19:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:19:16Z" name: isvc-secondary-8047bd-predictor.18a6dc25601eebe2 namespace: kserve-ci-e2e-test resourceVersion: "19206" uid: 76e675d4-fffb-41ef-8fc4-3c2da7cff961 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-8047bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "19050" uid: af33a606-4961-4b05-8b99-0db27897fff7 kind: Event lastTimestamp: "2026-04-16T14:19:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:19:16Z" name: isvc-secondary-8047bd-predictor.18a6dc25605f267c namespace: kserve-ci-e2e-test resourceVersion: "19208" uid: d33ab8bf-81cc-4ff3-b499-82b7e3ded7a9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-04-16T14:19:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-8047bd namespace: kserve-ci-e2e-test resourceVersion: "19040" uid: 3f11c3c2-6c8c-4f1c-98cb-2871c37521e7 kind: Event lastTimestamp: "2026-04-16T14:19:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:19:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:19:08Z" name: isvc-secondary-8047bd.18a6dc21e19071dd namespace: kserve-ci-e2e-test resourceVersion: "19147" uid: def6dc1a-bfbe-4f83-82f2-1a4948094ba2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-8047bd namespace: kserve-ci-e2e-test resourceVersion: "19040" uid: 3f11c3c2-6c8c-4f1c-98cb-2871c37521e7 kind: Event lastTimestamp: "2026-04-16T14:19:01Z" message: 'Failed to update status for InferenceService "isvc-secondary-8047bd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-8047bd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:19:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:19:01Z" name: isvc-secondary-8047bd.18a6dc21e23e366e namespace: kserve-ci-e2e-test resourceVersion: "19056" uid: dced24cc-6998-4f8a-b924-63ad23bf50c0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-8047bd namespace: kserve-ci-e2e-test resourceVersion: "19040" uid: 3f11c3c2-6c8c-4f1c-98cb-2871c37521e7 kind: Event lastTimestamp: "2026-04-16T14:19:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-8047bd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:19:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:19:01Z" name: isvc-secondary-8047bd.18a6dc21e23e720c namespace: kserve-ci-e2e-test resourceVersion: "19057" uid: f1455fa6-0ba5-4990-995b-b800e21107dd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:10:24.859305Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq namespace: kserve-ci-e2e-test resourceVersion: "14563" uid: 109a7777-da00-4d77-911a-aac714533f34 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq to ip-10-0-129-84.ec2.internal metadata: creationTimestamp: "2026-04-16T14:10:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:10:24Z" name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq.18a6dba9adfe31b4 namespace: kserve-ci-e2e-test resourceVersion: "14574" uid: f1880ab1-fe0b-457e-91db-34e40ad7abfe reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-86d7fd6df4-tkh2q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:25Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq namespace: kserve-ci-e2e-test resourceVersion: "14580" uid: 109a7777-da00-4d77-911a-aac714533f34 kind: Event lastTimestamp: "2026-04-16T14:10:25Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:10:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:10:25Z" name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq.18a6dba9c81e1751 namespace: kserve-ci-e2e-test resourceVersion: "14606" uid: e5e0541b-72a8-4ef4-880a-1b6423c035a0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq namespace: kserve-ci-e2e-test resourceVersion: "14569" uid: 109a7777-da00-4d77-911a-aac714533f34 kind: Event lastTimestamp: "2026-04-16T14:10:25Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8eee296dba42148fd03e4b2602ce20b8f3cc18bf537b500919938742ad784fb1" already present on machine metadata: creationTimestamp: "2026-04-16T14:10:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:25Z" name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq.18a6dba9c9c58a3f namespace: kserve-ci-e2e-test resourceVersion: "14608" uid: ded2788d-e8c0-4545-9e2d-0a341f5866e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq namespace: kserve-ci-e2e-test resourceVersion: "14569" uid: 109a7777-da00-4d77-911a-aac714533f34 kind: Event lastTimestamp: "2026-04-16T14:10:25Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:10:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:25Z" name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq.18a6dba9ce037228 namespace: kserve-ci-e2e-test resourceVersion: "14614" uid: 3e3ec73d-64f9-4375-b377-df58fddb63df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq namespace: kserve-ci-e2e-test resourceVersion: "14569" uid: 109a7777-da00-4d77-911a-aac714533f34 kind: Event lastTimestamp: "2026-04-16T14:10:25Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:10:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:25Z" name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq.18a6dba9ce70cc37 namespace: kserve-ci-e2e-test resourceVersion: "14616" uid: 8334f3f5-bbe4-4bbb-8ddc-d1df0e4b0eac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq namespace: kserve-ci-e2e-test resourceVersion: "14569" uid: 109a7777-da00-4d77-911a-aac714533f34 kind: Event lastTimestamp: "2026-04-16T14:10:29Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1424" already present on machine metadata: creationTimestamp: "2026-04-16T14:10:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:29Z" name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq.18a6dbaad9c61341 namespace: kserve-ci-e2e-test resourceVersion: "14661" uid: eb2ee458-d9f8-4239-85b9-565643f3835f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq namespace: kserve-ci-e2e-test resourceVersion: "14569" uid: 109a7777-da00-4d77-911a-aac714533f34 kind: Event lastTimestamp: "2026-04-16T14:10:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:10:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:29Z" name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq.18a6dbaadccaa786 namespace: kserve-ci-e2e-test resourceVersion: "14665" uid: aebc9de6-8c68-4b95-9bfd-9ca6b77c0fd7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq namespace: kserve-ci-e2e-test resourceVersion: "14569" uid: 109a7777-da00-4d77-911a-aac714533f34 kind: Event lastTimestamp: "2026-04-16T14:10:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:10:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:29Z" name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq.18a6dbaadd3d1f60 namespace: kserve-ci-e2e-test resourceVersion: "14667" uid: 43f6d54e-5cd9-4def-a3a5-9c5ca44cbf91 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T14:10:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq namespace: kserve-ci-e2e-test resourceVersion: "14569" uid: 109a7777-da00-4d77-911a-aac714533f34 kind: Event lastTimestamp: "2026-04-16T14:11:31Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T14:10:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:31Z" name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq.18a6dbab1610e49c namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: 1a308f0f-df2c-41aa-88b7-c7f05f2af319 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq namespace: kserve-ci-e2e-test resourceVersion: "14569" uid: 109a7777-da00-4d77-911a-aac714533f34 kind: Event lastTimestamp: "2026-04-16T14:12:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:12:05Z" name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq.18a6dbc106262c52 namespace: kserve-ci-e2e-test resourceVersion: "15500" uid: 3d28e200-bf0e-4297-8fd5-bc6f3a84f997 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9 namespace: kserve-ci-e2e-test resourceVersion: "14544" uid: c6919903-d536-4d6c-bcdd-fc936a0c37b4 kind: Event lastTimestamp: "2026-04-16T14:10:24Z" message: 'Created pod: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9-g6qhq' metadata: creationTimestamp: "2026-04-16T14:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:10:24Z" name: isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9.18a6dba9ad9b610e namespace: kserve-ci-e2e-test resourceVersion: "14568" uid: 21e321da-ff79-4b6b-be9d-471f1de27899 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-3af68-predictor namespace: kserve-ci-e2e-test resourceVersion: "14540" uid: 00b52ade-d379-403f-9b5c-19fbfcce42c6 kind: Event lastTimestamp: "2026-04-16T14:10:24Z" message: Scaled up replica set isvc-sklearn-graph-raw-3af68-predictor-ccc9568f9 from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:10:24Z" name: isvc-sklearn-graph-raw-3af68-predictor.18a6dba9a9767c53 namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: b535ad60-e9d1-4aa7-b1d6-4fb9d3ed8283 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:10:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-3af68-predictor namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 983ee4e3-bd23-4b89-a8d9-3f2d1bb19299 kind: Event lastTimestamp: "2026-04-16T14:11:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:10:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:11:09Z" name: isvc-sklearn-graph-raw-3af68-predictor.18a6dbad2841e649 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 50be1644-ea53-4877-88a7-1b2545a5521b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:10:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-3af68-predictor namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 983ee4e3-bd23-4b89-a8d9-3f2d1bb19299 kind: Event lastTimestamp: "2026-04-16T14:11:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:10:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:11:09Z" name: isvc-sklearn-graph-raw-3af68-predictor.18a6dbad28a347f0 namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: 3fcb3617-672a-4418-bbfc-f3a343bfa43d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:11:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-3af68-predictor namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: 983ee4e3-bd23-4b89-a8d9-3f2d1bb19299 kind: Event lastTimestamp: "2026-04-16T14:11:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:11:54Z" name: isvc-sklearn-graph-raw-3af68-predictor.18a6dbb7a5573c73 namespace: kserve-ci-e2e-test resourceVersion: "15328" uid: 8804a302-c01c-488c-9314-959aebf3430a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:11:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-3af68-predictor namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: 983ee4e3-bd23-4b89-a8d9-3f2d1bb19299 kind: Event lastTimestamp: "2026-04-16T14:11:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:11:54Z" name: isvc-sklearn-graph-raw-3af68-predictor.18a6dbb7a5b3ada5 namespace: kserve-ci-e2e-test resourceVersion: "15330" uid: 8a4651cd-dff0-4695-9c8d-60792c8871e1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T14:10:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-3af68 namespace: kserve-ci-e2e-test resourceVersion: "14517" uid: 6f5485e9-2477-4002-bcde-d20123c51579 kind: Event lastTimestamp: "2026-04-16T14:11:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:11:40Z" name: isvc-sklearn-graph-raw-3af68.18a6dba9a94eff0e namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 8c319163-35f8-45ef-9e1b-94d116d84b48 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-3af68 namespace: kserve-ci-e2e-test resourceVersion: "14517" uid: 6f5485e9-2477-4002-bcde-d20123c51579 kind: Event lastTimestamp: "2026-04-16T14:10:24Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-3af68": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-3af68": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:10:24Z" name: isvc-sklearn-graph-raw-3af68.18a6dba9a9b88bc7 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: ac8fdf8b-dd74-48ab-a24c-e739da4a7877 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-3af68 namespace: kserve-ci-e2e-test resourceVersion: "15219" uid: 6f5485e9-2477-4002-bcde-d20123c51579 kind: Event lastTimestamp: "2026-04-16T14:11:40Z" message: InferenceService [isvc-sklearn-graph-raw-3af68] is Ready metadata: creationTimestamp: "2026-04-16T14:11:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:11:40Z" name: isvc-sklearn-graph-raw-3af68.18a6dbbb5426ae16 namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: 357ac1d6-4d3e-46d3-b787-899101e6346f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:12:05.157904Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5 namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: 5826bce5-daea-4687-a04f-6ed05214e98d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5 to ip-10-0-129-84.ec2.internal metadata: creationTimestamp: "2026-04-16T14:12:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:12:05Z" name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5.18a6dbc108417283 namespace: kserve-ci-e2e-test resourceVersion: "15518" uid: b32d43e3-394a-4c75-a0d7-80d99f85f8c7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-86d7fd6df4-tkh2q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5 namespace: kserve-ci-e2e-test resourceVersion: "15527" uid: 5826bce5-daea-4687-a04f-6ed05214e98d kind: Event lastTimestamp: "2026-04-16T14:12:05Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:12:05Z" name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5.18a6dbc1220dee67 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: cfa0e3bc-9d13-480c-b328-729b8ca76982 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5 namespace: kserve-ci-e2e-test resourceVersion: "15514" uid: 5826bce5-daea-4687-a04f-6ed05214e98d kind: Event lastTimestamp: "2026-04-16T14:12:05Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8eee296dba42148fd03e4b2602ce20b8f3cc18bf537b500919938742ad784fb1" already present on machine metadata: creationTimestamp: "2026-04-16T14:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:12:05Z" name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5.18a6dbc123579e50 namespace: kserve-ci-e2e-test resourceVersion: "15569" uid: dceb1c61-ea42-4b76-aafa-2793cfb3379f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5 namespace: kserve-ci-e2e-test resourceVersion: "15514" uid: 5826bce5-daea-4687-a04f-6ed05214e98d kind: Event lastTimestamp: "2026-04-16T14:12:05Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:12:05Z" name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5.18a6dbc12642170f namespace: kserve-ci-e2e-test resourceVersion: "15572" uid: 08bbaa9f-216d-4f6a-8852-36707c8afc30 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5 namespace: kserve-ci-e2e-test resourceVersion: "15514" uid: 5826bce5-daea-4687-a04f-6ed05214e98d kind: Event lastTimestamp: "2026-04-16T14:12:05Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:12:05Z" name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5.18a6dbc126b535fa namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 1523db99-d8a2-43d6-a4dc-9870a9b69702 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5 namespace: kserve-ci-e2e-test resourceVersion: "15514" uid: 5826bce5-daea-4687-a04f-6ed05214e98d kind: Event lastTimestamp: "2026-04-16T14:12:09Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1424" already present on machine metadata: creationTimestamp: "2026-04-16T14:12:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:12:09Z" name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5.18a6dbc1fc8d9cd9 namespace: kserve-ci-e2e-test resourceVersion: "15618" uid: 3439831f-a72b-401b-8f43-b6dea8e0f709 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5 namespace: kserve-ci-e2e-test resourceVersion: "15514" uid: 5826bce5-daea-4687-a04f-6ed05214e98d kind: Event lastTimestamp: "2026-04-16T14:12:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:12:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:12:09Z" name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5.18a6dbc2001c8458 namespace: kserve-ci-e2e-test resourceVersion: "15624" uid: 03654b8e-85bd-4901-9006-ec530aefd5ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5 namespace: kserve-ci-e2e-test resourceVersion: "15514" uid: 5826bce5-daea-4687-a04f-6ed05214e98d kind: Event lastTimestamp: "2026-04-16T14:12:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:12:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:12:09Z" name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5.18a6dbc20097623a namespace: kserve-ci-e2e-test resourceVersion: "15625" uid: 3b635726-719f-420d-aba0-10cc7b8f363f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T14:12:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5 namespace: kserve-ci-e2e-test resourceVersion: "15514" uid: 5826bce5-daea-4687-a04f-6ed05214e98d kind: Event lastTimestamp: "2026-04-16T14:13:45Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T14:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:45Z" name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5.18a6dbc238c7fe4c namespace: kserve-ci-e2e-test resourceVersion: "16493" uid: 9836cbe0-6bf9-46a2-871d-3abe59435bc3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5 namespace: kserve-ci-e2e-test resourceVersion: "15514" uid: 5826bce5-daea-4687-a04f-6ed05214e98d kind: Event lastTimestamp: "2026-04-16T14:13:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:45Z" name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5.18a6dbd85fda440f namespace: kserve-ci-e2e-test resourceVersion: "16453" uid: 1e80544a-c470-450e-bf3f-9c8cfe7d6fe2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459 namespace: kserve-ci-e2e-test resourceVersion: "15479" uid: 9fe2514c-a51d-44b8-a2d5-237e90289abb kind: Event lastTimestamp: "2026-04-16T14:12:05Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459-d7ck5' metadata: creationTimestamp: "2026-04-16T14:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:12:05Z" name: isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459.18a6dbc107e80e51 namespace: kserve-ci-e2e-test resourceVersion: "15513" uid: ac30d132-5dbe-4cd3-a2f1-ffcb0ec8dea5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-3c9de-predictor namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: 0c5d4e3d-2985-4445-af58-bfed25423045 kind: Event lastTimestamp: "2026-04-16T14:12:05Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-3c9de-predictor-5c5b76c459 from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:12:05Z" name: isvc-sklearn-graph-raw-hpa-3c9de-predictor.18a6dbc103c4af30 namespace: kserve-ci-e2e-test resourceVersion: "15482" uid: bd76972e-2606-4054-9ef9-174e5105208e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3c9de-predictor namespace: kserve-ci-e2e-test resourceVersion: "15478" uid: ed4d11a1-a224-46aa-9130-7802e7077279 kind: Event lastTimestamp: "2026-04-16T14:12:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:12:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:12:20Z" name: isvc-sklearn-graph-raw-hpa-3c9de-predictor.18a6dbc48245ebe2 namespace: kserve-ci-e2e-test resourceVersion: "15717" uid: 77c2fc69-b46a-4fc2-9910-f94766dcde26 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3c9de-predictor namespace: kserve-ci-e2e-test resourceVersion: "15478" uid: ed4d11a1-a224-46aa-9130-7802e7077279 kind: Event lastTimestamp: "2026-04-16T14:12:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:12:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:12:20Z" name: isvc-sklearn-graph-raw-hpa-3c9de-predictor.18a6dbc482aa4048 namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: 93a0b48b-05db-4886-86e8-3f22c928addb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T14:12:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3c9de-predictor namespace: kserve-ci-e2e-test resourceVersion: "15718" uid: ed4d11a1-a224-46aa-9130-7802e7077279 kind: Event lastTimestamp: "2026-04-16T14:13:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:12:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:35Z" name: isvc-sklearn-graph-raw-hpa-3c9de-predictor.18a6dbc801a342ed namespace: kserve-ci-e2e-test resourceVersion: "16294" uid: 4832d609-773b-45e8-8ce0-95db71f899e5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T14:12:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3c9de-predictor namespace: kserve-ci-e2e-test resourceVersion: "15718" uid: ed4d11a1-a224-46aa-9130-7802e7077279 kind: Event lastTimestamp: "2026-04-16T14:13:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:12:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:35Z" name: isvc-sklearn-graph-raw-hpa-3c9de-predictor.18a6dbc80203836e namespace: kserve-ci-e2e-test resourceVersion: "16297" uid: 01bf5536-d0cb-4e5d-9e24-95e1d365a0c1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T14:12:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-3c9de namespace: kserve-ci-e2e-test resourceVersion: "15469" uid: 9188885b-7503-4c10-9a36-fda728108c8f kind: Event lastTimestamp: "2026-04-16T14:13:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:13:15Z" name: isvc-sklearn-graph-raw-hpa-3c9de.18a6dbc10398726b namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: 7537d393-9e6e-4c91-b120-3f3ef9abf7f1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-3c9de namespace: kserve-ci-e2e-test resourceVersion: "16146" uid: 9188885b-7503-4c10-9a36-fda728108c8f kind: Event lastTimestamp: "2026-04-16T14:13:15Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-3c9de] is Ready metadata: creationTimestamp: "2026-04-16T14:13:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:13:15Z" name: isvc-sklearn-graph-raw-hpa-3c9de.18a6dbd172499fa5 namespace: kserve-ci-e2e-test resourceVersion: "16147" uid: 9e421c19-0e22-4af2-a780-f49f29622300 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:15:10.764378Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf namespace: kserve-ci-e2e-test resourceVersion: "17255" uid: 5563408b-4cf9-4c8f-a535-d65c7d955862 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf to ip-10-0-129-84.ec2.internal metadata: creationTimestamp: "2026-04-16T14:15:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:15:10Z" name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf.18a6dbec3f437235 namespace: kserve-ci-e2e-test resourceVersion: "17260" uid: fff030df-27c1-4473-bfda-010766a6ad76 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-86d7fd6df4-tkh2q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:15:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf namespace: kserve-ci-e2e-test resourceVersion: "17263" uid: 5563408b-4cf9-4c8f-a535-d65c7d955862 kind: Event lastTimestamp: "2026-04-16T14:15:11Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:15:11Z" name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf.18a6dbec5851a557 namespace: kserve-ci-e2e-test resourceVersion: "17289" uid: 681ab461-ca3c-40aa-bb91-9a3cc25719ce reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf namespace: kserve-ci-e2e-test resourceVersion: "17257" uid: 5563408b-4cf9-4c8f-a535-d65c7d955862 kind: Event lastTimestamp: "2026-04-16T14:15:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8eee296dba42148fd03e4b2602ce20b8f3cc18bf537b500919938742ad784fb1" already present on machine metadata: creationTimestamp: "2026-04-16T14:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:15:11Z" name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf.18a6dbec598be4f2 namespace: kserve-ci-e2e-test resourceVersion: "17291" uid: e241fb1d-2e17-4c6a-adb7-7307a9ac87a3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf namespace: kserve-ci-e2e-test resourceVersion: "17257" uid: 5563408b-4cf9-4c8f-a535-d65c7d955862 kind: Event lastTimestamp: "2026-04-16T14:15:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:15:11Z" name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf.18a6dbec5cbf3a35 namespace: kserve-ci-e2e-test resourceVersion: "17293" uid: 77a518a7-9769-4f71-b5c1-f1749381adad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf namespace: kserve-ci-e2e-test resourceVersion: "17257" uid: 5563408b-4cf9-4c8f-a535-d65c7d955862 kind: Event lastTimestamp: "2026-04-16T14:15:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:15:11Z" name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf.18a6dbec5dcac670 namespace: kserve-ci-e2e-test resourceVersion: "17296" uid: cf980e60-c51b-472c-9994-5c48831b634f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf namespace: kserve-ci-e2e-test resourceVersion: "17257" uid: 5563408b-4cf9-4c8f-a535-d65c7d955862 kind: Event lastTimestamp: "2026-04-16T14:15:14Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1424" already present on machine metadata: creationTimestamp: "2026-04-16T14:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:15:14Z" name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf.18a6dbed376f6249 namespace: kserve-ci-e2e-test resourceVersion: "17327" uid: 1c5eb90d-0ce6-442d-aa88-e3ad5a59f593 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf namespace: kserve-ci-e2e-test resourceVersion: "17257" uid: 5563408b-4cf9-4c8f-a535-d65c7d955862 kind: Event lastTimestamp: "2026-04-16T14:15:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:15:14Z" name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf.18a6dbed3b301398 namespace: kserve-ci-e2e-test resourceVersion: "17332" uid: 6810cd46-5820-44ab-90dd-9b61d8d40339 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf namespace: kserve-ci-e2e-test resourceVersion: "17257" uid: 5563408b-4cf9-4c8f-a535-d65c7d955862 kind: Event lastTimestamp: "2026-04-16T14:15:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:15:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:15:14Z" name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf.18a6dbed3ba35bbb namespace: kserve-ci-e2e-test resourceVersion: "17333" uid: 328b8778-73b8-4250-ba42-d4f608851fba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 17 eventTime: null firstTimestamp: "2026-04-16T14:15:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf namespace: kserve-ci-e2e-test resourceVersion: "17257" uid: 5563408b-4cf9-4c8f-a535-d65c7d955862 kind: Event lastTimestamp: "2026-04-16T14:17:43Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T14:15:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:17:43Z" name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf.18a6dbed736e3795 namespace: kserve-ci-e2e-test resourceVersion: "18467" uid: 4f453c53-8106-4fde-ba91-251f25eae9f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf namespace: kserve-ci-e2e-test resourceVersion: "17257" uid: 5563408b-4cf9-4c8f-a535-d65c7d955862 kind: Event lastTimestamp: "2026-04-16T14:17:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:17:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:17:40Z" name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf.18a6dc0f383f7209 namespace: kserve-ci-e2e-test resourceVersion: "18411" uid: 62d30a41-73c8-4a22-b601-be268010b549 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:15:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9 namespace: kserve-ci-e2e-test resourceVersion: "17231" uid: cf890560-c9aa-4936-97ac-4e353e5e6def kind: Event lastTimestamp: "2026-04-16T14:15:10Z" message: 'Created pod: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9-cglzf' metadata: creationTimestamp: "2026-04-16T14:15:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:15:10Z" name: isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9.18a6dbec3ecc5c24 namespace: kserve-ci-e2e-test resourceVersion: "17256" uid: 786d4e79-5815-42b0-a572-171897a9d59c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:15:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-5c508-predictor namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: e7c8e204-9eb1-41f7-b600-4f132e2a7dc9 kind: Event lastTimestamp: "2026-04-16T14:15:10Z" message: Scaled up replica set isvc-sklearn-scale-raw-5c508-predictor-75cd64b9b9 from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:15:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:15:10Z" name: isvc-sklearn-scale-raw-5c508-predictor.18a6dbec3b8e004b namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: f284a4a8-0820-4420-90f4-8587fa2aaa60 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:15:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-5c508-predictor namespace: kserve-ci-e2e-test resourceVersion: "17230" uid: f3237ec2-5565-47c7-b94b-deb252605147 kind: Event lastTimestamp: "2026-04-16T14:15:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:15:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:15:55Z" name: isvc-sklearn-scale-raw-5c508-predictor.18a6dbefba08a4f3 namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 5f4fbe67-bd03-4585-affb-f456378f9c65 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:15:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-5c508-predictor namespace: kserve-ci-e2e-test resourceVersion: "17230" uid: f3237ec2-5565-47c7-b94b-deb252605147 kind: Event lastTimestamp: "2026-04-16T14:15:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:15:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:15:55Z" name: isvc-sklearn-scale-raw-5c508-predictor.18a6dbefba6c22d9 namespace: kserve-ci-e2e-test resourceVersion: "17632" uid: 03cfaea8-5c66-448f-88a6-bd9b143729d9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T14:16:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-5c508-predictor namespace: kserve-ci-e2e-test resourceVersion: "17413" uid: f3237ec2-5565-47c7-b94b-deb252605147 kind: Event lastTimestamp: "2026-04-16T14:17:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:16:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:17:40Z" name: isvc-sklearn-scale-raw-5c508-predictor.18a6dbfa374fec13 namespace: kserve-ci-e2e-test resourceVersion: "18378" uid: 1338f0e4-148b-4d4e-b241-bcabe62bb0f3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T14:16:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-5c508-predictor namespace: kserve-ci-e2e-test resourceVersion: "17413" uid: f3237ec2-5565-47c7-b94b-deb252605147 kind: Event lastTimestamp: "2026-04-16T14:17:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:16:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:17:40Z" name: isvc-sklearn-scale-raw-5c508-predictor.18a6dbfa37ba9a76 namespace: kserve-ci-e2e-test resourceVersion: "18379" uid: ad6beeb9-3df3-41ce-9fee-ac0174bab280 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T14:15:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-5c508 namespace: kserve-ci-e2e-test resourceVersion: "17216" uid: dc85c186-4ac6-49e6-86c4-402cbd3e2ccd kind: Event lastTimestamp: "2026-04-16T14:17:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:15:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:17:33Z" name: isvc-sklearn-scale-raw-5c508.18a6dbec3b5dc82b namespace: kserve-ci-e2e-test resourceVersion: "18322" uid: 1801d263-1c63-4544-a65a-cc020b0ae417 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:15:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-5c508 namespace: kserve-ci-e2e-test resourceVersion: "17216" uid: dc85c186-4ac6-49e6-86c4-402cbd3e2ccd kind: Event lastTimestamp: "2026-04-16T14:15:10Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-5c508": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-5c508": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:15:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:15:10Z" name: isvc-sklearn-scale-raw-5c508.18a6dbec3c4b5d5b namespace: kserve-ci-e2e-test resourceVersion: "17243" uid: 727c5c4d-d57f-4b72-9f95-6ece127163b0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:17:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-5c508 namespace: kserve-ci-e2e-test resourceVersion: "18320" uid: dc85c186-4ac6-49e6-86c4-402cbd3e2ccd kind: Event lastTimestamp: "2026-04-16T14:17:33Z" message: InferenceService [isvc-sklearn-scale-raw-5c508] is Ready metadata: creationTimestamp: "2026-04-16T14:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:17:33Z" name: isvc-sklearn-scale-raw-5c508.18a6dc0d8521c26f namespace: kserve-ci-e2e-test resourceVersion: "18321" uid: a99a97a2-ae04-4070-8b2e-8c27287d6760 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:10:24.903754Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7 namespace: kserve-ci-e2e-test resourceVersion: "14583" uid: 6c1cd6f5-74a2-47b0-b431-864185372950 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7 to ip-10-0-129-84.ec2.internal metadata: creationTimestamp: "2026-04-16T14:10:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:10:24Z" name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7.18a6dba9b0a4a9bd namespace: kserve-ci-e2e-test resourceVersion: "14589" uid: e214d720-ed2e-452d-b5b8-c5948be23bc6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-86d7fd6df4-tkh2q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:25Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7 namespace: kserve-ci-e2e-test resourceVersion: "14593" uid: 6c1cd6f5-74a2-47b0-b431-864185372950 kind: Event lastTimestamp: "2026-04-16T14:10:25Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:10:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:10:25Z" name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7.18a6dba9cadd282a namespace: kserve-ci-e2e-test resourceVersion: "14609" uid: f2cb0ed8-695c-49f6-b3f0-d192b83b5335 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7 namespace: kserve-ci-e2e-test resourceVersion: "14586" uid: 6c1cd6f5-74a2-47b0-b431-864185372950 kind: Event lastTimestamp: "2026-04-16T14:10:25Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8eee296dba42148fd03e4b2602ce20b8f3cc18bf537b500919938742ad784fb1" already present on machine metadata: creationTimestamp: "2026-04-16T14:10:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:25Z" name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7.18a6dba9cc28c247 namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: 8aa99af1-41fb-44e5-bee2-d6f11c7e13aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7 namespace: kserve-ci-e2e-test resourceVersion: "14586" uid: 6c1cd6f5-74a2-47b0-b431-864185372950 kind: Event lastTimestamp: "2026-04-16T14:10:25Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:10:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:25Z" name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7.18a6dba9cf0a38e9 namespace: kserve-ci-e2e-test resourceVersion: "14617" uid: cc6e8804-c174-4097-8cb6-3c31929106ac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7 namespace: kserve-ci-e2e-test resourceVersion: "14586" uid: 6c1cd6f5-74a2-47b0-b431-864185372950 kind: Event lastTimestamp: "2026-04-16T14:10:25Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:10:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:25Z" name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7.18a6dba9cf785d39 namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: ddc7847b-e6b3-43cf-9a75-1c027760404e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7 namespace: kserve-ci-e2e-test resourceVersion: "14586" uid: 6c1cd6f5-74a2-47b0-b431-864185372950 kind: Event lastTimestamp: "2026-04-16T14:10:29Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-16T14:10:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:29Z" name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7.18a6dbaad9f48b4d namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 13c7fcba-dd75-4a48-83c1-3104144052e6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7 namespace: kserve-ci-e2e-test resourceVersion: "14586" uid: 6c1cd6f5-74a2-47b0-b431-864185372950 kind: Event lastTimestamp: "2026-04-16T14:10:49Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.579s (19.579s including waiting). Image size: 1306229499 bytes.' metadata: creationTimestamp: "2026-04-16T14:10:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:49Z" name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7.18a6dbaf68fb8c43 namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: 12694d8f-6242-4413-ad56-73f547ebf28f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7 namespace: kserve-ci-e2e-test resourceVersion: "14586" uid: 6c1cd6f5-74a2-47b0-b431-864185372950 kind: Event lastTimestamp: "2026-04-16T14:10:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:10:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:49Z" name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7.18a6dbaf6ca2f9d3 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: 1067e3ed-07e3-4b08-99e4-28fd5f0ba881 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7 namespace: kserve-ci-e2e-test resourceVersion: "14586" uid: 6c1cd6f5-74a2-47b0-b431-864185372950 kind: Event lastTimestamp: "2026-04-16T14:10:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:10:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:10:49Z" name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7.18a6dbaf6d166e57 namespace: kserve-ci-e2e-test resourceVersion: "14831" uid: ed1c8e7b-5a80-425b-8759-db3bdae90a58 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T14:10:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7 namespace: kserve-ci-e2e-test resourceVersion: "14586" uid: 6c1cd6f5-74a2-47b0-b431-864185372950 kind: Event lastTimestamp: "2026-04-16T14:11:40Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T14:10:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:40Z" name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7.18a6dbaf86e5f3e4 namespace: kserve-ci-e2e-test resourceVersion: "15226" uid: c99bd347-6106-406b-8cb5-effde49a5755 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7 namespace: kserve-ci-e2e-test resourceVersion: "14586" uid: 6c1cd6f5-74a2-47b0-b431-864185372950 kind: Event lastTimestamp: "2026-04-16T14:12:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:12:05Z" name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7.18a6dbc1111bad27 namespace: kserve-ci-e2e-test resourceVersion: "15561" uid: 9127e753-8e22-4193-a936-9a4247dd17e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587 namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: 6f92cdb6-9155-446e-92f7-486751c5ad31 kind: Event lastTimestamp: "2026-04-16T14:10:24Z" message: 'Created pod: isvc-xgboost-graph-raw-3af68-predictor-55989b6587-vx9j7' metadata: creationTimestamp: "2026-04-16T14:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:10:24Z" name: isvc-xgboost-graph-raw-3af68-predictor-55989b6587.18a6dba9b03d5635 namespace: kserve-ci-e2e-test resourceVersion: "14585" uid: 86a56596-5e39-4d08-a87b-06dc0bcd786f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-3af68-predictor namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 4d7dfebd-57c0-4be2-9519-fab80782b97b kind: Event lastTimestamp: "2026-04-16T14:10:24Z" message: Scaled up replica set isvc-xgboost-graph-raw-3af68-predictor-55989b6587 from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:10:24Z" name: isvc-xgboost-graph-raw-3af68-predictor.18a6dba9ad5985ae namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 0741a4f5-a82c-4cd8-9ec6-313577b7fb7b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T14:10:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-3af68-predictor namespace: kserve-ci-e2e-test resourceVersion: "14561" uid: 71dfbeb3-f6e2-4d71-b857-121b9dee0004 kind: Event lastTimestamp: "2026-04-16T14:11:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:10:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:11:24Z" name: isvc-xgboost-graph-raw-3af68-predictor.18a6dbad2bbd6e25 namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: 12558244-10fe-46ab-bb96-d224c4b15500 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T14:10:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-3af68-predictor namespace: kserve-ci-e2e-test resourceVersion: "14561" uid: 71dfbeb3-f6e2-4d71-b857-121b9dee0004 kind: Event lastTimestamp: "2026-04-16T14:11:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:10:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:11:24Z" name: isvc-xgboost-graph-raw-3af68-predictor.18a6dbad2c197eaa namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: b3ec52aa-aa1e-43e6-8b90-6d94b4923622 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:11:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-3af68-predictor namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: 71dfbeb3-f6e2-4d71-b857-121b9dee0004 kind: Event lastTimestamp: "2026-04-16T14:11:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:11:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:11:54Z" name: isvc-xgboost-graph-raw-3af68-predictor.18a6dbbb26c505c8 namespace: kserve-ci-e2e-test resourceVersion: "15341" uid: 4b9914f2-e685-4598-9088-518754244f65 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:11:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-3af68-predictor namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: 71dfbeb3-f6e2-4d71-b857-121b9dee0004 kind: Event lastTimestamp: "2026-04-16T14:11:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:11:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:11:54Z" name: isvc-xgboost-graph-raw-3af68-predictor.18a6dbbb271af1d4 namespace: kserve-ci-e2e-test resourceVersion: "15344" uid: d93999ce-e3f1-4a6c-bf5c-32b9e4e7e040 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T14:10:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-3af68 namespace: kserve-ci-e2e-test resourceVersion: "14553" uid: 79a874c0-6d42-4936-8620-d9af6ad4588f kind: Event lastTimestamp: "2026-04-16T14:11:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:11:51Z" name: isvc-xgboost-graph-raw-3af68.18a6dba9ad30a84d namespace: kserve-ci-e2e-test resourceVersion: "15299" uid: 48da898c-96bc-4d79-ac55-348c4572cd17 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:10:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-3af68 namespace: kserve-ci-e2e-test resourceVersion: "14553" uid: 79a874c0-6d42-4936-8620-d9af6ad4588f kind: Event lastTimestamp: "2026-04-16T14:10:24Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-3af68": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-3af68": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:10:24Z" name: isvc-xgboost-graph-raw-3af68.18a6dba9ada36efc namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: fa9f00d1-7bf7-4b2f-b590-e25aa217c740 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-3af68 namespace: kserve-ci-e2e-test resourceVersion: "15297" uid: 79a874c0-6d42-4936-8620-d9af6ad4588f kind: Event lastTimestamp: "2026-04-16T14:11:51Z" message: InferenceService [isvc-xgboost-graph-raw-3af68] is Ready metadata: creationTimestamp: "2026-04-16T14:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:11:51Z" name: isvc-xgboost-graph-raw-3af68.18a6dbbdbe84a073 namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 0c5f77f5-c593-4497-8bd7-1275ad48a2e6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:12:05.207479Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: 5e8d959e-e715-418f-9a13-3ad1a3dd6954 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s to ip-10-0-129-84.ec2.internal metadata: creationTimestamp: "2026-04-16T14:12:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:12:05Z" name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s.18a6dbc10b35d0cf namespace: kserve-ci-e2e-test resourceVersion: "15537" uid: dee106b6-1b39-4bce-8dc0-9c063587f955 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-86d7fd6df4-tkh2q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s namespace: kserve-ci-e2e-test resourceVersion: "15542" uid: 5e8d959e-e715-418f-9a13-3ad1a3dd6954 kind: Event lastTimestamp: "2026-04-16T14:12:05Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:12:05Z" name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s.18a6dbc125468618 namespace: kserve-ci-e2e-test resourceVersion: "15570" uid: 714bfd8c-8d78-4119-9cc8-332fdefc09b8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s namespace: kserve-ci-e2e-test resourceVersion: "15532" uid: 5e8d959e-e715-418f-9a13-3ad1a3dd6954 kind: Event lastTimestamp: "2026-04-16T14:12:05Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8eee296dba42148fd03e4b2602ce20b8f3cc18bf537b500919938742ad784fb1" already present on machine metadata: creationTimestamp: "2026-04-16T14:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:12:05Z" name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s.18a6dbc1265c30df namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: fdeb54cb-138b-4ceb-8709-dc11bc36993f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s namespace: kserve-ci-e2e-test resourceVersion: "15532" uid: 5e8d959e-e715-418f-9a13-3ad1a3dd6954 kind: Event lastTimestamp: "2026-04-16T14:12:05Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:12:05Z" name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s.18a6dbc12a6eee3c namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: da865d0d-ed71-489f-b8c5-5dca1cd584e3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s namespace: kserve-ci-e2e-test resourceVersion: "15532" uid: 5e8d959e-e715-418f-9a13-3ad1a3dd6954 kind: Event lastTimestamp: "2026-04-16T14:12:05Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:12:05Z" name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s.18a6dbc12ae3abf1 namespace: kserve-ci-e2e-test resourceVersion: "15579" uid: d91e35a9-78b4-4d74-b6bb-33902e35cc9d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s namespace: kserve-ci-e2e-test resourceVersion: "15532" uid: 5e8d959e-e715-418f-9a13-3ad1a3dd6954 kind: Event lastTimestamp: "2026-04-16T14:12:10Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-16T14:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:12:10Z" name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s.18a6dbc2388e04ee namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: 242ff17c-21ab-464f-b035-c6aa7dcb85d2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s namespace: kserve-ci-e2e-test resourceVersion: "15532" uid: 5e8d959e-e715-418f-9a13-3ad1a3dd6954 kind: Event lastTimestamp: "2026-04-16T14:12:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:12:10Z" name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s.18a6dbc23c002695 namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: e7ccec9a-cfab-4471-810b-9516df2f1535 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s namespace: kserve-ci-e2e-test resourceVersion: "15532" uid: 5e8d959e-e715-418f-9a13-3ad1a3dd6954 kind: Event lastTimestamp: "2026-04-16T14:12:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:12:10Z" name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s.18a6dbc23c70e928 namespace: kserve-ci-e2e-test resourceVersion: "15645" uid: f18871fd-eaf6-4479-8c87-f39943fa5bb2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T14:12:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s namespace: kserve-ci-e2e-test resourceVersion: "15532" uid: 5e8d959e-e715-418f-9a13-3ad1a3dd6954 kind: Event lastTimestamp: "2026-04-16T14:13:02Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T14:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:02Z" name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s.18a6dbc274b50c21 namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: c3f62e08-61ba-4a00-afea-c3be187b310a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s namespace: kserve-ci-e2e-test resourceVersion: "15532" uid: 5e8d959e-e715-418f-9a13-3ad1a3dd6954 kind: Event lastTimestamp: "2026-04-16T14:13:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:45Z" name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s.18a6dbd8659ea510 namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: c40bae05-e7ea-41c2-8b5d-cd27e893867e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59 namespace: kserve-ci-e2e-test resourceVersion: "15516" uid: f85b0f55-b634-4d38-96b5-548e3ffe7866 kind: Event lastTimestamp: "2026-04-16T14:12:05Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59-pfb2s' metadata: creationTimestamp: "2026-04-16T14:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:12:05Z" name: isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59.18a6dbc10ab2c2c9 namespace: kserve-ci-e2e-test resourceVersion: "15531" uid: 9a7246c4-3e22-4541-9719-a6b32185a524 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-3c9de-predictor namespace: kserve-ci-e2e-test resourceVersion: "15511" uid: 7987d6ac-3065-4dd5-9506-165965b6f6f9 kind: Event lastTimestamp: "2026-04-16T14:12:05Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-3c9de-predictor-6948979d59 from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:12:05Z" name: isvc-xgboost-graph-raw-hpa-3c9de-predictor.18a6dbc108950f18 namespace: kserve-ci-e2e-test resourceVersion: "15522" uid: 09a61ab6-5c22-471f-87c8-a68334d07966 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3c9de-predictor namespace: kserve-ci-e2e-test resourceVersion: "15517" uid: 248b77a9-afdc-4431-8579-1afe33194577 kind: Event lastTimestamp: "2026-04-16T14:12:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:12:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:12:20Z" name: isvc-xgboost-graph-raw-hpa-3c9de-predictor.18a6dbc487513259 namespace: kserve-ci-e2e-test resourceVersion: "15720" uid: 1aa657d4-95b5-4f5d-97fe-e2554647df0f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3c9de-predictor namespace: kserve-ci-e2e-test resourceVersion: "15517" uid: 248b77a9-afdc-4431-8579-1afe33194577 kind: Event lastTimestamp: "2026-04-16T14:12:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:12:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:12:20Z" name: isvc-xgboost-graph-raw-hpa-3c9de-predictor.18a6dbc487b5480c namespace: kserve-ci-e2e-test resourceVersion: "15722" uid: 2721d6b0-8426-4590-8892-387d5c470135 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T14:12:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3c9de-predictor namespace: kserve-ci-e2e-test resourceVersion: "15721" uid: 248b77a9-afdc-4431-8579-1afe33194577 kind: Event lastTimestamp: "2026-04-16T14:13:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:12:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:35Z" name: isvc-xgboost-graph-raw-hpa-3c9de-predictor.18a6dbc80656aa3d namespace: kserve-ci-e2e-test resourceVersion: "16313" uid: f061f448-2880-48af-bb83-dc39f8bcb98f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T14:12:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3c9de-predictor namespace: kserve-ci-e2e-test resourceVersion: "15721" uid: 248b77a9-afdc-4431-8579-1afe33194577 kind: Event lastTimestamp: "2026-04-16T14:13:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:12:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:35Z" name: isvc-xgboost-graph-raw-hpa-3c9de-predictor.18a6dbc806b6e0fa namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: f650ba5e-2dfb-4219-a20b-6f8a8711a3e0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T14:12:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-3c9de namespace: kserve-ci-e2e-test resourceVersion: "15498" uid: c3fcd4db-1008-48e2-9223-7d15bccc1922 kind: Event lastTimestamp: "2026-04-16T14:13:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:13:12Z" name: isvc-xgboost-graph-raw-hpa-3c9de.18a6dbc108999315 namespace: kserve-ci-e2e-test resourceVersion: "16112" uid: 57095160-ecd9-4fec-93ba-5cec0921a083 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-3c9de namespace: kserve-ci-e2e-test resourceVersion: "16109" uid: c3fcd4db-1008-48e2-9223-7d15bccc1922 kind: Event lastTimestamp: "2026-04-16T14:13:12Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-3c9de] is Ready metadata: creationTimestamp: "2026-04-16T14:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:13:12Z" name: isvc-xgboost-graph-raw-hpa-3c9de.18a6dbd0ac714b3d namespace: kserve-ci-e2e-test resourceVersion: "16110" uid: e0683fb6-9250-4bde-baff-907d232fff60 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:13:45.427873Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-a0d01-predictor-6455c7c84d-kslv2 namespace: kserve-ci-e2e-test resourceVersion: "16455" uid: 41c93375-a90d-4947-88f3-a086a4c482f7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-a0d01-predictor-6455c7c84d-kslv2 to ip-10-0-130-98.ec2.internal metadata: creationTimestamp: "2026-04-16T14:13:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:13:45Z" name: message-dumper-raw-a0d01-predictor-6455c7c84d-kslv2.18a6dbd860cfa6ea namespace: kserve-ci-e2e-test resourceVersion: "16461" uid: f9049165-dcce-4da8-9a6a-fcdcfac3ecf1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-86d7fd6df4-tkh2q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:45Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-a0d01-predictor-6455c7c84d-kslv2 namespace: kserve-ci-e2e-test resourceVersion: "16466" uid: 41c93375-a90d-4947-88f3-a086a4c482f7 kind: Event lastTimestamp: "2026-04-16T14:13:45Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:13:45Z" name: message-dumper-raw-a0d01-predictor-6455c7c84d-kslv2.18a6dbd868cbaee6 namespace: kserve-ci-e2e-test resourceVersion: "16490" uid: 63dc8b05-4711-4cc5-bd36-d52c461e012e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-a0d01-predictor-6455c7c84d-kslv2 namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: 41c93375-a90d-4947-88f3-a086a4c482f7 kind: Event lastTimestamp: "2026-04-16T14:13:45Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-16T14:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:45Z" name: message-dumper-raw-a0d01-predictor-6455c7c84d-kslv2.18a6dbd869ed971b namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: 5f80ceb3-ad7d-4515-9086-8b13b31e75f2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-98.ec2.internal source: component: kubelet host: ip-10-0-130-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-a0d01-predictor-6455c7c84d-kslv2 namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: 41c93375-a90d-4947-88f3-a086a4c482f7 kind: Event lastTimestamp: "2026-04-16T14:13:46Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 997ms (997ms including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-16T14:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:46Z" name: message-dumper-raw-a0d01-predictor-6455c7c84d-kslv2.18a6dbd8a56657a8 namespace: kserve-ci-e2e-test resourceVersion: "16501" uid: 59630abf-85d0-463a-ae71-569d8647ffb4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-98.ec2.internal source: component: kubelet host: ip-10-0-130-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-a0d01-predictor-6455c7c84d-kslv2 namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: 41c93375-a90d-4947-88f3-a086a4c482f7 kind: Event lastTimestamp: "2026-04-16T14:13:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:46Z" name: message-dumper-raw-a0d01-predictor-6455c7c84d-kslv2.18a6dbd8a91bcdd3 namespace: kserve-ci-e2e-test resourceVersion: "16502" uid: 4937cffe-1822-4c44-ad38-519f23394142 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-98.ec2.internal source: component: kubelet host: ip-10-0-130-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-a0d01-predictor-6455c7c84d-kslv2 namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: 41c93375-a90d-4947-88f3-a086a4c482f7 kind: Event lastTimestamp: "2026-04-16T14:13:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:46Z" name: message-dumper-raw-a0d01-predictor-6455c7c84d-kslv2.18a6dbd8a98a6df2 namespace: kserve-ci-e2e-test resourceVersion: "16503" uid: 2b7e1457-7b4c-4f17-ab08-dcc61bd38782 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-98.ec2.internal source: component: kubelet host: ip-10-0-130-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:15:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-a0d01-predictor-6455c7c84d-kslv2 namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: 41c93375-a90d-4947-88f3-a086a4c482f7 kind: Event lastTimestamp: "2026-04-16T14:15:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:15:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:15:10Z" name: message-dumper-raw-a0d01-predictor-6455c7c84d-kslv2.18a6dbec42cd95b3 namespace: kserve-ci-e2e-test resourceVersion: "17280" uid: 00799db6-adcb-467a-be47-bbb966dd78d0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-98.ec2.internal source: component: kubelet host: ip-10-0-130-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-a0d01-predictor-6455c7c84d namespace: kserve-ci-e2e-test resourceVersion: "16433" uid: cd7cffb1-710f-455b-990c-747242c59e96 kind: Event lastTimestamp: "2026-04-16T14:13:45Z" message: 'Created pod: message-dumper-raw-a0d01-predictor-6455c7c84d-kslv2' metadata: creationTimestamp: "2026-04-16T14:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:45Z" name: message-dumper-raw-a0d01-predictor-6455c7c84d.18a6dbd860801fde namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 6b196092-6eeb-435c-840c-732e3cb030de reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-a0d01-predictor namespace: kserve-ci-e2e-test resourceVersion: "16430" uid: 23afc509-e55d-4209-b1ae-85f90b1abc0f kind: Event lastTimestamp: "2026-04-16T14:13:45Z" message: Scaled up replica set message-dumper-raw-a0d01-predictor-6455c7c84d from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:45Z" name: message-dumper-raw-a0d01-predictor.18a6dbd85d93af8c namespace: kserve-ci-e2e-test resourceVersion: "16437" uid: 6d8e40a5-423f-4bc5-b3d5-eedc83daec79 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:14:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-a0d01-predictor namespace: kserve-ci-e2e-test resourceVersion: "16432" uid: a5f8166c-848f-4859-b4b6-7f0f4f55f24b kind: Event lastTimestamp: "2026-04-16T14:14:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:14:15Z" name: message-dumper-raw-a0d01-predictor.18a6dbdbdc5663b0 namespace: kserve-ci-e2e-test resourceVersion: "16803" uid: 737a4919-016f-4c48-b785-29ac50e083ea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:14:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-a0d01-predictor namespace: kserve-ci-e2e-test resourceVersion: "16432" uid: a5f8166c-848f-4859-b4b6-7f0f4f55f24b kind: Event lastTimestamp: "2026-04-16T14:14:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:14:15Z" name: message-dumper-raw-a0d01-predictor.18a6dbdbdd0bd88f namespace: kserve-ci-e2e-test resourceVersion: "16804" uid: fcadf76c-8d7b-4577-8ecd-d78ca985a1b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T14:13:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-a0d01 namespace: kserve-ci-e2e-test resourceVersion: "16424" uid: 7db0d65a-3c17-42a0-8561-0226c2a9922a kind: Event lastTimestamp: "2026-04-16T14:13:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:13:46Z" name: message-dumper-raw-a0d01.18a6dbd85d61e061 namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 4ab6958b-1aac-45fc-a27b-4abaf41ff92e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-a0d01 namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: 7db0d65a-3c17-42a0-8561-0226c2a9922a kind: Event lastTimestamp: "2026-04-16T14:13:45Z" message: 'Failed to update status for InferenceService "message-dumper-raw-a0d01": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-a0d01": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:13:45Z" name: message-dumper-raw-a0d01.18a6dbd8605b2794 namespace: kserve-ci-e2e-test resourceVersion: "16456" uid: 185e195a-78f6-4ba8-92b3-a8a0a206295a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-a0d01 namespace: kserve-ci-e2e-test resourceVersion: "16515" uid: 7db0d65a-3c17-42a0-8561-0226c2a9922a kind: Event lastTimestamp: "2026-04-16T14:13:46Z" message: InferenceService [message-dumper-raw-a0d01] is Ready metadata: creationTimestamp: "2026-04-16T14:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:13:46Z" name: message-dumper-raw-a0d01.18a6dbd8b772285b namespace: kserve-ci-e2e-test resourceVersion: "16516" uid: 4bc08a87-f2e5-41a7-bb8a-b31bb339a3a6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:11:54.935283Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-3af68-fffcc59fc-72kql namespace: kserve-ci-e2e-test resourceVersion: "15342" uid: ec026471-7a50-4527-bade-8ca250ab1679 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-3af68-fffcc59fc-72kql to ip-10-0-130-98.ec2.internal metadata: creationTimestamp: "2026-04-16T14:11:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:11:54Z" name: model-chainer-raw-3af68-fffcc59fc-72kql.18a6dbbea6f08c03 namespace: kserve-ci-e2e-test resourceVersion: "15348" uid: ae55687e-b248-49e7-bd84-2ac5eadeae1b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-86d7fd6df4-tkh2q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:55Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-3af68-fffcc59fc-72kql namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: ec026471-7a50-4527-bade-8ca250ab1679 kind: Event lastTimestamp: "2026-04-16T14:11:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-3af68-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T14:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:55Z" name: model-chainer-raw-3af68-fffcc59fc-72kql.18a6dbbeadc9b8f8 namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 37a6df3c-e3b0-4ba7-9cb7-cc9a49bf9f13 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-98.ec2.internal source: component: kubelet host: ip-10-0-130-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:55Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-3af68-fffcc59fc-72kql namespace: kserve-ci-e2e-test resourceVersion: "15352" uid: ec026471-7a50-4527-bade-8ca250ab1679 kind: Event lastTimestamp: "2026-04-16T14:11:55Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:11:55Z" name: model-chainer-raw-3af68-fffcc59fc-72kql.18a6dbbee3a84c98 namespace: kserve-ci-e2e-test resourceVersion: "15366" uid: 3774bf1d-f35e-4dd6-934c-ac9fac1b117e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-3af68} kind: Pod name: model-chainer-raw-3af68-fffcc59fc-72kql namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: ec026471-7a50-4527-bade-8ca250ab1679 kind: Event lastTimestamp: "2026-04-16T14:11:55Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:24e0d07fc928f25bef5abad5b7d6f0433a913fef03e0557ec5bebf00bc2024ca" metadata: creationTimestamp: "2026-04-16T14:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:55Z" name: model-chainer-raw-3af68-fffcc59fc-72kql.18a6dbbee50b5129 namespace: kserve-ci-e2e-test resourceVersion: "15368" uid: aa087d03-06b2-4117-88b8-331bb0a29c08 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-98.ec2.internal source: component: kubelet host: ip-10-0-130-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-3af68} kind: Pod name: model-chainer-raw-3af68-fffcc59fc-72kql namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: ec026471-7a50-4527-bade-8ca250ab1679 kind: Event lastTimestamp: "2026-04-16T14:11:58Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:24e0d07fc928f25bef5abad5b7d6f0433a913fef03e0557ec5bebf00bc2024ca" in 2.207s (2.207s including waiting). Image size: 216230490 bytes.' metadata: creationTimestamp: "2026-04-16T14:11:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:58Z" name: model-chainer-raw-3af68-fffcc59fc-72kql.18a6dbbf68a5a1e0 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 79f1bb9c-5314-4e0a-97ed-6c24b77f71e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-98.ec2.internal source: component: kubelet host: ip-10-0-130-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-3af68} kind: Pod name: model-chainer-raw-3af68-fffcc59fc-72kql namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: ec026471-7a50-4527-bade-8ca250ab1679 kind: Event lastTimestamp: "2026-04-16T14:11:58Z" message: 'Created container: model-chainer-raw-3af68' metadata: creationTimestamp: "2026-04-16T14:11:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:58Z" name: model-chainer-raw-3af68-fffcc59fc-72kql.18a6dbbf6baa66e7 namespace: kserve-ci-e2e-test resourceVersion: "15389" uid: 9d93f9aa-1670-4957-afea-2b54976c9cf0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-98.ec2.internal source: component: kubelet host: ip-10-0-130-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-3af68} kind: Pod name: model-chainer-raw-3af68-fffcc59fc-72kql namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: ec026471-7a50-4527-bade-8ca250ab1679 kind: Event lastTimestamp: "2026-04-16T14:11:58Z" message: Started container model-chainer-raw-3af68 metadata: creationTimestamp: "2026-04-16T14:11:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:11:58Z" name: model-chainer-raw-3af68-fffcc59fc-72kql.18a6dbbf6c2f702a namespace: kserve-ci-e2e-test resourceVersion: "15390" uid: 00c73eaa-8630-42bb-9bf7-d40415e77d02 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-98.ec2.internal source: component: kubelet host: ip-10-0-130-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:12:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-3af68} kind: Pod name: model-chainer-raw-3af68-fffcc59fc-72kql namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: ec026471-7a50-4527-bade-8ca250ab1679 kind: Event lastTimestamp: "2026-04-16T14:12:04Z" message: Stopping container model-chainer-raw-3af68 metadata: creationTimestamp: "2026-04-16T14:12:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:12:04Z" name: model-chainer-raw-3af68-fffcc59fc-72kql.18a6dbc0fd9b3026 namespace: kserve-ci-e2e-test resourceVersion: "15464" uid: 7dce2cf6-ad28-42e6-805c-cad56c4fa591 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-98.ec2.internal source: component: kubelet host: ip-10-0-130-98.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T14:12:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-3af68} kind: Pod name: model-chainer-raw-3af68-fffcc59fc-72kql namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: ec026471-7a50-4527-bade-8ca250ab1679 kind: Event lastTimestamp: "2026-04-16T14:12:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T14:12:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:12:34Z" name: model-chainer-raw-3af68-fffcc59fc-72kql.18a6dbc20a21048b namespace: kserve-ci-e2e-test resourceVersion: "15829" uid: e2fb93d6-84c1-486e-8139-d3a14e0e1255 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-98.ec2.internal source: component: kubelet host: ip-10-0-130-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-3af68-fffcc59fc namespace: kserve-ci-e2e-test resourceVersion: "15337" uid: 250d7d9d-8513-46fc-bb8b-5c3ff5244cb9 kind: Event lastTimestamp: "2026-04-16T14:11:54Z" message: 'Created pod: model-chainer-raw-3af68-fffcc59fc-72kql' metadata: creationTimestamp: "2026-04-16T14:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:11:54Z" name: model-chainer-raw-3af68-fffcc59fc.18a6dbbea690f0de namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: 8b6f484a-4eeb-49c9-8510-4c7ebe1b2e92 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-3af68 namespace: kserve-ci-e2e-test resourceVersion: "15336" uid: 15bc8c3a-8df8-4002-9de7-54e70015c685 kind: Event lastTimestamp: "2026-04-16T14:11:54Z" message: Scaled up replica set model-chainer-raw-3af68-fffcc59fc from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:11:54Z" name: model-chainer-raw-3af68.18a6dbbea5c34e67 namespace: kserve-ci-e2e-test resourceVersion: "15339" uid: 64bb27f3-e1ce-4a8f-bc3a-83195248a12e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:11:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-3af68 namespace: kserve-ci-e2e-test resourceVersion: "15354" uid: 7b4e400d-a76e-4f04-ba8c-b2d0d5a3e0fe kind: Event lastTimestamp: "2026-04-16T14:11:55Z" message: InferenceGraph [model-chainer-raw-3af68] is Ready metadata: creationTimestamp: "2026-04-16T14:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:11:55Z" name: model-chainer-raw-3af68.18a6dbbeac94c8cc namespace: kserve-ci-e2e-test resourceVersion: "15356" uid: 8508cd78-3eba-4b85-bf33-7620b56dabe9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:13:35.207343Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-3c9de-5fd8d44b5c-jfd5m namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: faf44163-4949-45b9-aa49-f266586451eb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-3c9de-5fd8d44b5c-jfd5m to ip-10-0-130-98.ec2.internal metadata: creationTimestamp: "2026-04-16T14:13:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:13:35Z" name: model-chainer-raw-hpa-3c9de-5fd8d44b5c-jfd5m.18a6dbd5ff9ec73a namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: 1e980784-9656-4270-838b-ccae31db565d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-86d7fd6df4-tkh2q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:35Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-3c9de-5fd8d44b5c-jfd5m namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: faf44163-4949-45b9-aa49-f266586451eb kind: Event lastTimestamp: "2026-04-16T14:13:35Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:13:35Z" name: model-chainer-raw-hpa-3c9de-5fd8d44b5c-jfd5m.18a6dbd6189f10cf namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: f091b027-5e4d-4df5-9ca4-3e1a779a93d0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3c9de} kind: Pod name: model-chainer-raw-hpa-3c9de-5fd8d44b5c-jfd5m namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: faf44163-4949-45b9-aa49-f266586451eb kind: Event lastTimestamp: "2026-04-16T14:13:35Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24e0d07fc928f25bef5abad5b7d6f0433a913fef03e0557ec5bebf00bc2024ca" already present on machine metadata: creationTimestamp: "2026-04-16T14:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:35Z" name: model-chainer-raw-hpa-3c9de-5fd8d44b5c-jfd5m.18a6dbd619dd9b3b namespace: kserve-ci-e2e-test resourceVersion: "16323" uid: 32fb4736-ceea-4e3c-8491-93f23d7fbac8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-98.ec2.internal source: component: kubelet host: ip-10-0-130-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3c9de} kind: Pod name: model-chainer-raw-hpa-3c9de-5fd8d44b5c-jfd5m namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: faf44163-4949-45b9-aa49-f266586451eb kind: Event lastTimestamp: "2026-04-16T14:13:35Z" message: 'Created container: model-chainer-raw-hpa-3c9de' metadata: creationTimestamp: "2026-04-16T14:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:35Z" name: model-chainer-raw-hpa-3c9de-5fd8d44b5c-jfd5m.18a6dbd61d0c155b namespace: kserve-ci-e2e-test resourceVersion: "16324" uid: 0b01c730-3e86-41c8-8f48-c74d256d64f5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-98.ec2.internal source: component: kubelet host: ip-10-0-130-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3c9de} kind: Pod name: model-chainer-raw-hpa-3c9de-5fd8d44b5c-jfd5m namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: faf44163-4949-45b9-aa49-f266586451eb kind: Event lastTimestamp: "2026-04-16T14:13:35Z" message: Started container model-chainer-raw-hpa-3c9de metadata: creationTimestamp: "2026-04-16T14:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:35Z" name: model-chainer-raw-hpa-3c9de-5fd8d44b5c-jfd5m.18a6dbd61d6a5be6 namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: 4b3af67a-2ac8-4b6d-8c98-ee17c30f5f26 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-98.ec2.internal source: component: kubelet host: ip-10-0-130-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3c9de} kind: Pod name: model-chainer-raw-hpa-3c9de-5fd8d44b5c-jfd5m namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: faf44163-4949-45b9-aa49-f266586451eb kind: Event lastTimestamp: "2026-04-16T14:13:45Z" message: Stopping container model-chainer-raw-hpa-3c9de metadata: creationTimestamp: "2026-04-16T14:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:13:45Z" name: model-chainer-raw-hpa-3c9de-5fd8d44b5c-jfd5m.18a6dbd85774bbed namespace: kserve-ci-e2e-test resourceVersion: "16420" uid: 03646172-eeaa-4771-857c-10396963f721 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-98.ec2.internal source: component: kubelet host: ip-10-0-130-98.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T14:13:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3c9de} kind: Pod name: model-chainer-raw-hpa-3c9de-5fd8d44b5c-jfd5m namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: faf44163-4949-45b9-aa49-f266586451eb kind: Event lastTimestamp: "2026-04-16T14:14:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T14:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:14:11Z" name: model-chainer-raw-hpa-3c9de-5fd8d44b5c-jfd5m.18a6dbd8b198ab55 namespace: kserve-ci-e2e-test resourceVersion: "16776" uid: 3b93bed2-0d44-4325-a5a5-a2e26aab7855 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-98.ec2.internal source: component: kubelet host: ip-10-0-130-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-3c9de-5fd8d44b5c namespace: kserve-ci-e2e-test resourceVersion: "16296" uid: 52e74ceb-6cc0-49e5-bd6f-4ba27ee46c85 kind: Event lastTimestamp: "2026-04-16T14:13:35Z" message: 'Created pod: model-chainer-raw-hpa-3c9de-5fd8d44b5c-jfd5m' metadata: creationTimestamp: "2026-04-16T14:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:35Z" name: model-chainer-raw-hpa-3c9de-5fd8d44b5c.18a6dbd5ff1451c2 namespace: kserve-ci-e2e-test resourceVersion: "16302" uid: a62a2239-ca33-4df0-8fc1-c1ce35223360 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-3c9de namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: 4002458c-9420-4127-b6ca-f4feb39af921 kind: Event lastTimestamp: "2026-04-16T14:13:35Z" message: Scaled up replica set model-chainer-raw-hpa-3c9de-5fd8d44b5c from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:13:35Z" name: model-chainer-raw-hpa-3c9de.18a6dbd5fe3ae225 namespace: kserve-ci-e2e-test resourceVersion: "16298" uid: 550659e7-65b1-4b17-abe2-6379e95d4ae7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:13:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-3c9de namespace: kserve-ci-e2e-test resourceVersion: "16314" uid: 6f8dba7e-ab16-4176-8b55-edd841cc33be kind: Event lastTimestamp: "2026-04-16T14:13:35Z" message: InferenceGraph [model-chainer-raw-hpa-3c9de] is Ready metadata: creationTimestamp: "2026-04-16T14:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:13:35Z" name: model-chainer-raw-hpa-3c9de.18a6dbd60231be95 namespace: kserve-ci-e2e-test resourceVersion: "16318" uid: 278ce57c-d14f-409f-9194-a87d4e18e830 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:19:27.504821Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t namespace: kserve-ci-e2e-test resourceVersion: "19432" uid: 8d46a32f-58c9-49ca-9727-6efb2317fda9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t to ip-10-0-129-84.ec2.internal metadata: creationTimestamp: "2026-04-16T14:19:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:19:27Z" name: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t.18a6dc28062fa333 namespace: kserve-ci-e2e-test resourceVersion: "19438" uid: 202a43e3-bca5-48e4-b32e-56001b22b5d0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-86d7fd6df4-tkh2q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:27Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t namespace: kserve-ci-e2e-test resourceVersion: "19443" uid: 8d46a32f-58c9-49ca-9727-6efb2317fda9 kind: Event lastTimestamp: "2026-04-16T14:19:27Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:19:27Z" name: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t.18a6dc281fada6b6 namespace: kserve-ci-e2e-test resourceVersion: "19454" uid: cae6fccd-dfd1-4d57-abc6-803d1b7cd490 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: 8d46a32f-58c9-49ca-9727-6efb2317fda9 kind: Event lastTimestamp: "2026-04-16T14:19:27Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8eee296dba42148fd03e4b2602ce20b8f3cc18bf537b500919938742ad784fb1" already present on machine metadata: creationTimestamp: "2026-04-16T14:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:27Z" name: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t.18a6dc282143ab7a namespace: kserve-ci-e2e-test resourceVersion: "19457" uid: e8225bbf-9a4a-426f-88e0-0951ab851211 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: 8d46a32f-58c9-49ca-9727-6efb2317fda9 kind: Event lastTimestamp: "2026-04-16T14:19:28Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:28Z" name: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t.18a6dc282582b947 namespace: kserve-ci-e2e-test resourceVersion: "19460" uid: 6908432a-260c-4803-971c-1b7213d0cd7f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: 8d46a32f-58c9-49ca-9727-6efb2317fda9 kind: Event lastTimestamp: "2026-04-16T14:19:28Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:28Z" name: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t.18a6dc2825eee01a namespace: kserve-ci-e2e-test resourceVersion: "19461" uid: 17b3ba7b-f858-4835-848c-96989573bb94 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: 8d46a32f-58c9-49ca-9727-6efb2317fda9 kind: Event lastTimestamp: "2026-04-16T14:19:32Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1424" already present on machine metadata: creationTimestamp: "2026-04-16T14:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:32Z" name: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t.18a6dc2943aa8b4b namespace: kserve-ci-e2e-test resourceVersion: "19506" uid: 63f03394-bc5a-45c0-a607-284ede427ba0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: 8d46a32f-58c9-49ca-9727-6efb2317fda9 kind: Event lastTimestamp: "2026-04-16T14:19:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:32Z" name: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t.18a6dc2946d63690 namespace: kserve-ci-e2e-test resourceVersion: "19510" uid: 728a813c-2b3f-4d6e-9dbd-4ff7d4844483 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: 8d46a32f-58c9-49ca-9727-6efb2317fda9 kind: Event lastTimestamp: "2026-04-16T14:19:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:19:32Z" name: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t.18a6dc29474a59ee namespace: kserve-ci-e2e-test resourceVersion: "19511" uid: 522e0ebd-0bd2-40e6-ad74-fb1bcd1807e7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T14:19:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: 8d46a32f-58c9-49ca-9727-6efb2317fda9 kind: Event lastTimestamp: "2026-04-16T14:20:34Z" message: 'Readiness probe failed: dial tcp 10.132.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T14:19:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:20:34Z" name: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t.18a6dc297f9f118c namespace: kserve-ci-e2e-test resourceVersion: "19948" uid: d68db862-14b1-4590-b608-9701aa97aca2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: 8d46a32f-58c9-49ca-9727-6efb2317fda9 kind: Event lastTimestamp: "2026-04-16T14:20:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:20:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:20:47Z" name: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t.18a6dc3ab17467b7 namespace: kserve-ci-e2e-test resourceVersion: "20076" uid: 522599fd-59c7-4241-b125-76ce9837247a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-38bb0-predictor-5dbd6487cf namespace: kserve-ci-e2e-test resourceVersion: "19425" uid: 50e69549-18a5-4ee5-a755-e221aa5922dd kind: Event lastTimestamp: "2026-04-16T14:19:27Z" message: 'Created pod: raw-sklearn-38bb0-predictor-5dbd6487cf-grh4t' metadata: creationTimestamp: "2026-04-16T14:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:19:27Z" name: raw-sklearn-38bb0-predictor-5dbd6487cf.18a6dc28059833c1 namespace: kserve-ci-e2e-test resourceVersion: "19434" uid: 1cb62dc1-c606-4027-8250-f2361bc346f0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-38bb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "19422" uid: fbaa4b58-d498-4f05-9bd9-9e513d629136 kind: Event lastTimestamp: "2026-04-16T14:19:27Z" message: Scaled up replica set raw-sklearn-38bb0-predictor-5dbd6487cf from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:19:27Z" name: raw-sklearn-38bb0-predictor.18a6dc2803ba389c namespace: kserve-ci-e2e-test resourceVersion: "19427" uid: f9bfe36e-91b6-4224-b349-ab65d2a3f39f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:19:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-38bb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "19424" uid: 63a84353-73da-44c3-8d37-830cf8fa5393 kind: Event lastTimestamp: "2026-04-16T14:20:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:20:12Z" name: raw-sklearn-38bb0-predictor.18a6dc2b821e7e94 namespace: kserve-ci-e2e-test resourceVersion: "19794" uid: bc2f13d6-fcdd-41af-ae63-4b7d2dd6341a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:19:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-38bb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "19424" uid: 63a84353-73da-44c3-8d37-830cf8fa5393 kind: Event lastTimestamp: "2026-04-16T14:20:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:20:12Z" name: raw-sklearn-38bb0-predictor.18a6dc2b827de740 namespace: kserve-ci-e2e-test resourceVersion: "19795" uid: 441da71b-a763-4b3b-8221-7569845eada3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:20:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-38bb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "19580" uid: 63a84353-73da-44c3-8d37-830cf8fa5393 kind: Event lastTimestamp: "2026-04-16T14:20:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:20:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:20:42Z" name: raw-sklearn-38bb0-predictor.18a6dc35ffb2993f namespace: kserve-ci-e2e-test resourceVersion: "20004" uid: db7b0c5a-e1c0-46c8-b8fb-e65eefde0789 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:20:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-38bb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "19580" uid: 63a84353-73da-44c3-8d37-830cf8fa5393 kind: Event lastTimestamp: "2026-04-16T14:20:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:20:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:20:42Z" name: raw-sklearn-38bb0-predictor.18a6dc36001071f2 namespace: kserve-ci-e2e-test resourceVersion: "20005" uid: cfc8840d-afa6-448c-a7ce-2bd61954ed53 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T14:19:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-38bb0 namespace: kserve-ci-e2e-test resourceVersion: "19414" uid: e00bc72f-60b1-46e5-b7c3-ec43000586b3 kind: Event lastTimestamp: "2026-04-16T14:20:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:20:43Z" name: raw-sklearn-38bb0.18a6dc280377cf8a namespace: kserve-ci-e2e-test resourceVersion: "20022" uid: f6ff95ff-070d-4e47-a054-d6a300f476a2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:19:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-38bb0 namespace: kserve-ci-e2e-test resourceVersion: "19414" uid: e00bc72f-60b1-46e5-b7c3-ec43000586b3 kind: Event lastTimestamp: "2026-04-16T14:19:27Z" message: 'Failed to update status for InferenceService "raw-sklearn-38bb0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-38bb0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:19:27Z" name: raw-sklearn-38bb0.18a6dc2803eda789 namespace: kserve-ci-e2e-test resourceVersion: "19429" uid: 0ce8041a-37c5-4430-ab0a-62e2bc6aa019 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-38bb0 namespace: kserve-ci-e2e-test resourceVersion: "20020" uid: e00bc72f-60b1-46e5-b7c3-ec43000586b3 kind: Event lastTimestamp: "2026-04-16T14:20:43Z" message: InferenceService [raw-sklearn-38bb0] is Ready metadata: creationTimestamp: "2026-04-16T14:20:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:20:43Z" name: raw-sklearn-38bb0.18a6dc39c141d168 namespace: kserve-ci-e2e-test resourceVersion: "20021" uid: 857061cc-a5a7-4b49-9fde-eede3a76459d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T14:20:47.754498Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf namespace: kserve-ci-e2e-test resourceVersion: "20087" uid: 6e89da89-9d77-4803-8992-78f3861653ea kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf to ip-10-0-129-84.ec2.internal metadata: creationTimestamp: "2026-04-16T14:20:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T14:20:47Z" name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf.18a6dc3ab5708c84 namespace: kserve-ci-e2e-test resourceVersion: "20092" uid: 8fd709e2-9b53-4c58-a974-e84ff244eba9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-86d7fd6df4-tkh2q source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:48Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf namespace: kserve-ci-e2e-test resourceVersion: "20098" uid: 6e89da89-9d77-4803-8992-78f3861653ea kind: Event lastTimestamp: "2026-04-16T14:20:48Z" message: Add eth0 [10.132.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T14:20:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T14:20:48Z" name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf.18a6dc3acedce187 namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 93e3e60f-927b-41ef-942c-5040ef89653f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: 6e89da89-9d77-4803-8992-78f3861653ea kind: Event lastTimestamp: "2026-04-16T14:20:48Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8eee296dba42148fd03e4b2602ce20b8f3cc18bf537b500919938742ad784fb1" already present on machine metadata: creationTimestamp: "2026-04-16T14:20:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:20:48Z" name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf.18a6dc3ad0131768 namespace: kserve-ci-e2e-test resourceVersion: "20109" uid: c3df6cce-e886-4f2a-a2cc-4b898bedef8e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: 6e89da89-9d77-4803-8992-78f3861653ea kind: Event lastTimestamp: "2026-04-16T14:20:48Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T14:20:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:20:48Z" name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf.18a6dc3ad3634cab namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: 00bd8a54-42d5-42f3-9282-676946faeb0c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: 6e89da89-9d77-4803-8992-78f3861653ea kind: Event lastTimestamp: "2026-04-16T14:20:48Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T14:20:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:20:48Z" name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf.18a6dc3ad3ea2973 namespace: kserve-ci-e2e-test resourceVersion: "20113" uid: bf60e123-5594-44fc-865d-16babc633aa5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: 6e89da89-9d77-4803-8992-78f3861653ea kind: Event lastTimestamp: "2026-04-16T14:20:52Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1424" already present on machine metadata: creationTimestamp: "2026-04-16T14:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:20:52Z" name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf.18a6dc3bb92fe43b namespace: kserve-ci-e2e-test resourceVersion: "20144" uid: 1f5f0c79-4d18-4b43-b337-c56e98cd93c1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: 6e89da89-9d77-4803-8992-78f3861653ea kind: Event lastTimestamp: "2026-04-16T14:20:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T14:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:20:52Z" name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf.18a6dc3bbc94afed namespace: kserve-ci-e2e-test resourceVersion: "20150" uid: dbd75316-fa55-40e0-a92d-2b2e4bddab3f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: 6e89da89-9d77-4803-8992-78f3861653ea kind: Event lastTimestamp: "2026-04-16T14:20:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T14:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:20:52Z" name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf.18a6dc3bbd02b268 namespace: kserve-ci-e2e-test resourceVersion: "20151" uid: 55e3e8af-a591-4470-8690-73b7d92486b4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T14:20:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: 6e89da89-9d77-4803-8992-78f3861653ea kind: Event lastTimestamp: "2026-04-16T14:22:10Z" message: 'Readiness probe failed: dial tcp 10.132.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T14:20:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:22:10Z" name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf.18a6dc3bf53cb846 namespace: kserve-ci-e2e-test resourceVersion: "20762" uid: de823af3-83f3-4950-9496-2fa52a0d86d8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: 6e89da89-9d77-4803-8992-78f3861653ea kind: Event lastTimestamp: "2026-04-16T14:22:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T14:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T14:22:07Z" name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf.18a6dc4d5bd02f33 namespace: kserve-ci-e2e-test resourceVersion: "20713" uid: 40bc55c7-784c-46db-af2a-c5aeffcbdba6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-84.ec2.internal source: component: kubelet host: ip-10-0-129-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc namespace: kserve-ci-e2e-test resourceVersion: "20078" uid: 66b26863-e6b6-4c20-8e3d-8c7edcdaf4ab kind: Event lastTimestamp: "2026-04-16T14:20:47Z" message: 'Created pod: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc-gczrf' metadata: creationTimestamp: "2026-04-16T14:20:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:20:47Z" name: raw-sklearn-runtime-2c1fa-predictor-8ff766fcc.18a6dc3ab500b125 namespace: kserve-ci-e2e-test resourceVersion: "20089" uid: 9e906748-fc6f-4067-b185-7bcaa51afa52 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-2c1fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "20074" uid: a43037ab-4d56-4d24-b2d2-2533a8ce2001 kind: Event lastTimestamp: "2026-04-16T14:20:47Z" message: Scaled up replica set raw-sklearn-runtime-2c1fa-predictor-8ff766fcc from 0 to 1 metadata: creationTimestamp: "2026-04-16T14:20:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:20:47Z" name: raw-sklearn-runtime-2c1fa-predictor.18a6dc3ab228d467 namespace: kserve-ci-e2e-test resourceVersion: "20080" uid: 9907cee6-192a-4763-8d2d-4110dd48edee reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:21:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-2c1fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "20077" uid: 32cacff2-3b73-48d8-9c3a-bce313f60943 kind: Event lastTimestamp: "2026-04-16T14:21:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:21:17Z" name: raw-sklearn-runtime-2c1fa-predictor.18a6dc3e30fa53f2 namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: b28885dc-bbd8-4cf1-a8bb-97d5515141a6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T14:21:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-2c1fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "20077" uid: 32cacff2-3b73-48d8-9c3a-bce313f60943 kind: Event lastTimestamp: "2026-04-16T14:21:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T14:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:21:17Z" name: raw-sklearn-runtime-2c1fa-predictor.18a6dc3e315c1b33 namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: e02b2a2f-fbcb-4df7-a8c4-79843b34a0cd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:21:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-2c1fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "20227" uid: 32cacff2-3b73-48d8-9c3a-bce313f60943 kind: Event lastTimestamp: "2026-04-16T14:22:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:22:02Z" name: raw-sklearn-runtime-2c1fa-predictor.18a6dc452f380397 namespace: kserve-ci-e2e-test resourceVersion: "20661" uid: b0be9c5d-a224-4b40-ac63-559ed5a14660 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T14:21:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-2c1fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "20227" uid: 32cacff2-3b73-48d8-9c3a-bce313f60943 kind: Event lastTimestamp: "2026-04-16T14:22:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T14:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T14:22:02Z" name: raw-sklearn-runtime-2c1fa-predictor.18a6dc452f97725c namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 9b626410-8852-4dbf-a47c-7bfd2ceb6c5d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T14:20:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-2c1fa namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: 3a5b637a-57ca-4e36-8b5c-d1ba2e420d94 kind: Event lastTimestamp: "2026-04-16T14:22:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T14:20:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:22:00Z" name: raw-sklearn-runtime-2c1fa.18a6dc3ab2039db2 namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: 69bdec4e-7efd-46a8-9f49-575f01dc589b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:20:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-2c1fa namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: 3a5b637a-57ca-4e36-8b5c-d1ba2e420d94 kind: Event lastTimestamp: "2026-04-16T14:20:47Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-2c1fa": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-2c1fa": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T14:20:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:20:47Z" name: raw-sklearn-runtime-2c1fa.18a6dc3ab2654b99 namespace: kserve-ci-e2e-test resourceVersion: "20082" uid: 8614c0f9-19c4-4eb9-ae80-e19da300c748 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T14:22:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-2c1fa namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: 3a5b637a-57ca-4e36-8b5c-d1ba2e420d94 kind: Event lastTimestamp: "2026-04-16T14:22:00Z" message: InferenceService [raw-sklearn-runtime-2c1fa] is Ready metadata: creationTimestamp: "2026-04-16T14:22:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T14:22:00Z" name: raw-sklearn-runtime-2c1fa.18a6dc4baf36e70a namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: 735d181a-cb2a-40f7-9036-69ceb432ad7b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "20898"