--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:19:14.885856Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-3dec36-predictor-6f8b45dcff-p5629 namespace: kserve-ci-e2e-test resourceVersion: "18816" uid: 81669d50-52be-4ca4-98e5-7d3de6d4e7cc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-3dec36-predictor-6f8b45dcff-p5629 to ip-10-0-134-19.ec2.internal metadata: creationTimestamp: "2026-05-26T16:19:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:19:14Z" name: isvc-init-fail-3dec36-predictor-6f8b45dcff-p5629.18b329e82bd2c644 namespace: kserve-ci-e2e-test resourceVersion: "18821" uid: bb84db79-3c54-4adc-ade9-f20271a0f7c1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8b5dfd5d8-s8tjj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:15Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-3dec36-predictor-6f8b45dcff-p5629 namespace: kserve-ci-e2e-test resourceVersion: "18828" uid: 81669d50-52be-4ca4-98e5-7d3de6d4e7cc kind: Event lastTimestamp: "2026-05-26T16:19:15Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:19:15Z" name: isvc-init-fail-3dec36-predictor-6f8b45dcff-p5629.18b329e845757e3c namespace: kserve-ci-e2e-test resourceVersion: "18839" uid: 982320ec-0168-47b8-b575-a3cf55ace9d9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:19:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-3dec36-predictor-6f8b45dcff-p5629 namespace: kserve-ci-e2e-test resourceVersion: "18818" uid: 81669d50-52be-4ca4-98e5-7d3de6d4e7cc kind: Event lastTimestamp: "2026-05-26T16:19:21Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0413900e48a4ba69f75c45b06af88f792921f4669631dcf3a047c1fe5c1fc341" already present on machine metadata: creationTimestamp: "2026-05-26T16:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:19:21Z" name: isvc-init-fail-3dec36-predictor-6f8b45dcff-p5629.18b329e8479b7049 namespace: kserve-ci-e2e-test resourceVersion: "18898" uid: dbf2f488-ac52-41c6-9197-26785baa0e71 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:19:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-3dec36-predictor-6f8b45dcff-p5629 namespace: kserve-ci-e2e-test resourceVersion: "18818" uid: 81669d50-52be-4ca4-98e5-7d3de6d4e7cc kind: Event lastTimestamp: "2026-05-26T16:19:21Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T16:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:19:21Z" name: isvc-init-fail-3dec36-predictor-6f8b45dcff-p5629.18b329e84a6daf75 namespace: kserve-ci-e2e-test resourceVersion: "18900" uid: 96d3d18f-66bb-49d4-9afc-33ff2eaa29d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:19:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-3dec36-predictor-6f8b45dcff-p5629 namespace: kserve-ci-e2e-test resourceVersion: "18818" uid: 81669d50-52be-4ca4-98e5-7d3de6d4e7cc kind: Event lastTimestamp: "2026-05-26T16:19:21Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T16:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:19:21Z" name: isvc-init-fail-3dec36-predictor-6f8b45dcff-p5629.18b329e84b15bb79 namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: c5ff0a47-7a49-40b3-9eff-366d47a4815b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-3dec36-predictor-6f8b45dcff-p5629 namespace: kserve-ci-e2e-test resourceVersion: "18818" uid: 81669d50-52be-4ca4-98e5-7d3de6d4e7cc kind: Event lastTimestamp: "2026-05-26T16:19:24Z" message: Back-off restarting failed container storage-initializer in pod isvc-init-fail-3dec36-predictor-6f8b45dcff-p5629_kserve-ci-e2e-test(81669d50-52be-4ca4-98e5-7d3de6d4e7cc) metadata: creationTimestamp: "2026-05-26T16:19:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:19:24Z" name: isvc-init-fail-3dec36-predictor-6f8b45dcff-p5629.18b329ea5e40269b namespace: kserve-ci-e2e-test resourceVersion: "18933" uid: c63e9916-61d4-423b-8362-4a41fe7a0b06 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-3dec36-predictor-6f8b45dcff namespace: kserve-ci-e2e-test resourceVersion: "18793" uid: 1dedaa09-c40f-48a9-ba7b-7328fc18131d kind: Event lastTimestamp: "2026-05-26T16:19:14Z" message: 'Created pod: isvc-init-fail-3dec36-predictor-6f8b45dcff-p5629' metadata: creationTimestamp: "2026-05-26T16:19:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:19:14Z" name: isvc-init-fail-3dec36-predictor-6f8b45dcff.18b329e82b39fd81 namespace: kserve-ci-e2e-test resourceVersion: "18817" uid: e61c908f-f46d-4e43-812a-59392c82e223 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-3dec36-predictor namespace: kserve-ci-e2e-test resourceVersion: "18788" uid: 9852f17e-31f4-4848-9be8-9c6c2d71da69 kind: Event lastTimestamp: "2026-05-26T16:19:14Z" message: Scaled up replica set isvc-init-fail-3dec36-predictor-6f8b45dcff from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:19:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:19:14Z" name: isvc-init-fail-3dec36-predictor.18b329e82744a822 namespace: kserve-ci-e2e-test resourceVersion: "18799" uid: 8181a568-7fb6-4404-895b-7f3072d10250 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-05-26T16:19:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-3dec36 namespace: kserve-ci-e2e-test resourceVersion: "18765" uid: b95e29c4-2704-4e38-ae9a-9c87f15eda82 kind: Event lastTimestamp: "2026-05-26T16:19:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:19:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:19:24Z" name: isvc-init-fail-3dec36.18b329e82712d70c namespace: kserve-ci-e2e-test resourceVersion: "18937" uid: e8ca8ed9-7e47-48f1-a04d-eb459e282ff6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-3dec36 namespace: kserve-ci-e2e-test resourceVersion: "18798" uid: b95e29c4-2704-4e38-ae9a-9c87f15eda82 kind: Event lastTimestamp: "2026-05-26T16:19:14Z" message: 'Failed to update status for InferenceService "isvc-init-fail-3dec36": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-3dec36": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:19:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:19:14Z" name: isvc-init-fail-3dec36.18b329e82be783cc namespace: kserve-ci-e2e-test resourceVersion: "18824" uid: d6536915-293d-4e5d-ba83-e82c159bf3d6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-3dec36 namespace: kserve-ci-e2e-test resourceVersion: "18798" uid: b95e29c4-2704-4e38-ae9a-9c87f15eda82 kind: Event lastTimestamp: "2026-05-26T16:19:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-3dec36": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:19:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:19:14Z" name: isvc-init-fail-3dec36.18b329e82be7cbe9 namespace: kserve-ci-e2e-test resourceVersion: "18827" uid: 7ad8fe6b-c98c-406e-ac9b-81e2b4ec8f02 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:13:50.935171Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16117" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 to ip-10-0-134-19.ec2.internal metadata: creationTimestamp: "2026-05-26T16:13:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:13:50Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b3299cbedb9816 namespace: kserve-ci-e2e-test resourceVersion: "16121" uid: f5fa2eda-97c3-49ed-9f0d-abdc22d61a42 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8b5dfd5d8-s8tjj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: "2026-05-26T16:13:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-raw-4b426-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:51Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b3299cc68e9522 namespace: kserve-ci-e2e-test resourceVersion: "16129" uid: f3fd1d86-b3ff-4ead-9c89-b88ac00aa6f3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16125" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: "2026-05-26T16:13:51Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:13:51Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b3299cfc4fe371 namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: fe16b374-1451-4f54-8652-1c20ffa381bc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: "2026-05-26T16:13:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0413900e48a4ba69f75c45b06af88f792921f4669631dcf3a047c1fe5c1fc341" already present on machine metadata: creationTimestamp: "2026-05-26T16:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:52Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b3299cfe5be62b namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: 7d11b1c9-2a0a-4a5f-9422-3d317b02fa0e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: "2026-05-26T16:13:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T16:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:52Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b3299d00ddc2e8 namespace: kserve-ci-e2e-test resourceVersion: "16140" uid: 4e026304-ce56-4f4d-a600-6e1e601058c0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: "2026-05-26T16:13:52Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T16:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:52Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b3299d0148352e namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: 749d0862-7a1d-4799-b87a-d79ef9c198c0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: "2026-05-26T16:13:56Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:56Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b3299dfac926be namespace: kserve-ci-e2e-test resourceVersion: "16180" uid: 335ce31c-74e0-474b-910a-f17934ed9aca reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: "2026-05-26T16:13:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:56Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b3299dfedee43a namespace: kserve-ci-e2e-test resourceVersion: "16183" uid: 658a97ce-bce2-4035-a0d1-8259ed4aff9a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: "2026-05-26T16:13:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:56Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b3299dff5217b2 namespace: kserve-ci-e2e-test resourceVersion: "16186" uid: 066edf27-5c3d-49de-80ed-efdd4ec42411 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: "2026-05-26T16:13:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:56Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b3299dff5e768f namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: a5bc75ae-7d7b-4ea3-8cd7-288631394870 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: "2026-05-26T16:13:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:56Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b3299e022f7666 namespace: kserve-ci-e2e-test resourceVersion: "16188" uid: 87a302ee-390c-4056-b4a7-fa984d73e2b0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: "2026-05-26T16:13:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:56Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b3299e029e2c94 namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: 859e2b59-a185-4e4c-ab64-ddca033107f1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: "2026-05-26T16:13:56Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:d4dfc83be5dd58d8739a157278436e5a923b7626716bcc5e76e3c331a25671b6" already present on machine metadata: creationTimestamp: "2026-05-26T16:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:56Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b3299e02a92129 namespace: kserve-ci-e2e-test resourceVersion: "16191" uid: 83a394e0-c3e7-4082-8523-306ccddc38b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: "2026-05-26T16:13:56Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-05-26T16:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:56Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b3299e055c7fee namespace: kserve-ci-e2e-test resourceVersion: "16193" uid: 7ac7d9e4-4d4c-4d3e-b29d-404df666d4e5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: "2026-05-26T16:13:56Z" message: Started container agent metadata: creationTimestamp: "2026-05-26T16:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:56Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b3299e05c7fb3b namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: aadb1b38-7a79-4847-a0d0-053a3e2d202e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-26T16:13:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: "2026-05-26T16:15:33Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:13:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:33Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b3299e36d1ba47 namespace: kserve-ci-e2e-test resourceVersion: "17013" uid: 4beb7e90-3f05-415f-b36f-d116193e1d31 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-26T16:13:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: "2026-05-26T16:15:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T16:13:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:33Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b3299e36db2c26 namespace: kserve-ci-e2e-test resourceVersion: "17014" uid: f23a6c76-b3a9-4c4d-8a29-f2fe137b3b7d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: "2026-05-26T16:15:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:16Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b329b0962ae7d4 namespace: kserve-ci-e2e-test resourceVersion: "16818" uid: dc518f7d-6628-4958-af5f-5caaa4711efd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: "2026-05-26T16:15:16Z" message: Stopping container agent metadata: creationTimestamp: "2026-05-26T16:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:16Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b329b0962b2334 namespace: kserve-ci-e2e-test resourceVersion: "16819" uid: 71e8251a-67a3-4168-8d26-05155c5e8ec9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: "2026-05-26T16:15:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:16Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b329b0962b4bbd namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 38016ec3-1988-4bcf-b615-0d16b537d5fc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T16:15:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: a4bf19f8-f39c-4e8b-b38e-f26a45667f3c kind: Event lastTimestamp: "2026-05-26T16:15:33Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:15:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:33Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7.18b329b112ea12ce namespace: kserve-ci-e2e-test resourceVersion: "17012" uid: fca1336e-21f7-4e8a-8a2c-7341ef9a5dab reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-4b426-predictor-759bcfd9b namespace: kserve-ci-e2e-test resourceVersion: "16106" uid: fe90b758-9049-4c5a-8722-17517d2bd376 kind: Event lastTimestamp: "2026-05-26T16:13:50Z" message: 'Created pod: isvc-logger-raw-4b426-predictor-759bcfd9b-5qmg7' metadata: creationTimestamp: "2026-05-26T16:13:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:13:50Z" name: isvc-logger-raw-4b426-predictor-759bcfd9b.18b3299cbe3d675d namespace: kserve-ci-e2e-test resourceVersion: "16118" uid: 35e8e9d0-9be3-475a-af99-2371cbcc7f20 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-4b426-predictor namespace: kserve-ci-e2e-test resourceVersion: "16104" uid: 4654c6b9-a1a1-4d4e-b85d-ff91cfc3865e kind: Event lastTimestamp: "2026-05-26T16:13:50Z" message: Scaled up replica set isvc-logger-raw-4b426-predictor-759bcfd9b from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:13:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:13:50Z" name: isvc-logger-raw-4b426-predictor.18b3299cba6bfa54 namespace: kserve-ci-e2e-test resourceVersion: "16109" uid: ed55acb8-9e14-4e20-97dc-124f08aa4c6e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:14:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-4b426-predictor namespace: kserve-ci-e2e-test resourceVersion: "16105" uid: 87d08f6c-03b5-4007-baf0-3aab4fd122d4 kind: Event lastTimestamp: "2026-05-26T16:14:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:14:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:14:35Z" name: isvc-logger-raw-4b426-predictor.18b329a039158663 namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: bb608c70-d319-44fa-a0c0-bbde4ab6411a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:14:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-4b426-predictor namespace: kserve-ci-e2e-test resourceVersion: "16105" uid: 87d08f6c-03b5-4007-baf0-3aab4fd122d4 kind: Event lastTimestamp: "2026-05-26T16:14:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:14:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:14:35Z" name: isvc-logger-raw-4b426-predictor.18b329a039917da4 namespace: kserve-ci-e2e-test resourceVersion: "16484" uid: 97d4cc81-9bf9-4eec-9768-ec9bfff3eddb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:14:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-4b426-predictor namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 87d08f6c-03b5-4007-baf0-3aab4fd122d4 kind: Event lastTimestamp: "2026-05-26T16:15:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:15:05Z" name: isvc-logger-raw-4b426-predictor.18b329aab637fd37 namespace: kserve-ci-e2e-test resourceVersion: "16709" uid: b62347ea-2c70-4d8b-bb66-cb1511236b05 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:14:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-4b426-predictor namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 87d08f6c-03b5-4007-baf0-3aab4fd122d4 kind: Event lastTimestamp: "2026-05-26T16:15:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:15:05Z" name: isvc-logger-raw-4b426-predictor.18b329aab6b5e173 namespace: kserve-ci-e2e-test resourceVersion: "16710" uid: 02cfcfe1-0671-4efc-9a4f-3da3602985c0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-05-26T16:13:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-4b426 namespace: kserve-ci-e2e-test resourceVersion: "16094" uid: 3e168f9b-605e-4990-8d93-daf9a90e3d37 kind: Event lastTimestamp: "2026-05-26T16:15:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:13:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:15:03Z" name: isvc-logger-raw-4b426.18b3299cba432422 namespace: kserve-ci-e2e-test resourceVersion: "16693" uid: 015ecbfe-3a01-48b8-8bcc-e352f21a31e8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-4b426 namespace: kserve-ci-e2e-test resourceVersion: "16094" uid: 3e168f9b-605e-4990-8d93-daf9a90e3d37 kind: Event lastTimestamp: "2026-05-26T16:13:50Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-4b426": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-4b426": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:13:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:13:50Z" name: isvc-logger-raw-4b426.18b3299cbaf9ec5a namespace: kserve-ci-e2e-test resourceVersion: "16112" uid: d7ea4166-ab19-4752-9405-db789791e753 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-4b426 namespace: kserve-ci-e2e-test resourceVersion: "16690" uid: 3e168f9b-605e-4990-8d93-daf9a90e3d37 kind: Event lastTimestamp: "2026-05-26T16:15:03Z" message: InferenceService [isvc-logger-raw-4b426] is Ready metadata: creationTimestamp: "2026-05-26T16:15:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:15:03Z" name: isvc-logger-raw-4b426.18b329ad9b8f1e4c namespace: kserve-ci-e2e-test resourceVersion: "16691" uid: 5c02c816-2ed3-4da4-9f41-c08445654747 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:17:36.567634Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-ba2514-predictor-686dd7994-vcx9l namespace: kserve-ci-e2e-test resourceVersion: "17939" uid: 59ce4742-c702-4783-aca6-e7b3ab74513f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-ba2514-predictor-686dd7994-vcx9l to ip-10-0-134-19.ec2.internal metadata: creationTimestamp: "2026-05-26T16:17:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:17:36Z" name: isvc-primary-ba2514-predictor-686dd7994-vcx9l.18b329d14799c3a9 namespace: kserve-ci-e2e-test resourceVersion: "17945" uid: f248036f-bf54-4fd1-9caa-9ec194f12a78 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8b5dfd5d8-s8tjj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:37Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-ba2514-predictor-686dd7994-vcx9l namespace: kserve-ci-e2e-test resourceVersion: "17944" uid: 59ce4742-c702-4783-aca6-e7b3ab74513f kind: Event lastTimestamp: "2026-05-26T16:17:37Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:17:37Z" name: isvc-primary-ba2514-predictor-686dd7994-vcx9l.18b329d195e12e18 namespace: kserve-ci-e2e-test resourceVersion: "17964" uid: d373171d-896d-475f-ad90-2911b6a91273 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-ba2514-predictor-686dd7994-vcx9l namespace: kserve-ci-e2e-test resourceVersion: "17942" uid: 59ce4742-c702-4783-aca6-e7b3ab74513f kind: Event lastTimestamp: "2026-05-26T16:17:37Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0413900e48a4ba69f75c45b06af88f792921f4669631dcf3a047c1fe5c1fc341" already present on machine metadata: creationTimestamp: "2026-05-26T16:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:37Z" name: isvc-primary-ba2514-predictor-686dd7994-vcx9l.18b329d197a183ae namespace: kserve-ci-e2e-test resourceVersion: "17966" uid: ba0a64d4-6cc2-44bd-ab09-81889c8635f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-ba2514-predictor-686dd7994-vcx9l namespace: kserve-ci-e2e-test resourceVersion: "17942" uid: 59ce4742-c702-4783-aca6-e7b3ab74513f kind: Event lastTimestamp: "2026-05-26T16:17:37Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T16:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:37Z" name: isvc-primary-ba2514-predictor-686dd7994-vcx9l.18b329d19adce000 namespace: kserve-ci-e2e-test resourceVersion: "17967" uid: acceebfc-6688-4bb4-bd3c-8de47d3db244 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-ba2514-predictor-686dd7994-vcx9l namespace: kserve-ci-e2e-test resourceVersion: "17942" uid: 59ce4742-c702-4783-aca6-e7b3ab74513f kind: Event lastTimestamp: "2026-05-26T16:17:37Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T16:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:37Z" name: isvc-primary-ba2514-predictor-686dd7994-vcx9l.18b329d19b484255 namespace: kserve-ci-e2e-test resourceVersion: "17968" uid: c71e36a3-6706-42ae-973b-3b1b3f852ea5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-ba2514-predictor-686dd7994-vcx9l namespace: kserve-ci-e2e-test resourceVersion: "17942" uid: 59ce4742-c702-4783-aca6-e7b3ab74513f kind: Event lastTimestamp: "2026-05-26T16:17:41Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:17:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:41Z" name: isvc-primary-ba2514-predictor-686dd7994-vcx9l.18b329d2898c2141 namespace: kserve-ci-e2e-test resourceVersion: "18009" uid: 8afb2241-c42a-462e-8684-279c0d74a6d5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-ba2514-predictor-686dd7994-vcx9l namespace: kserve-ci-e2e-test resourceVersion: "17942" uid: 59ce4742-c702-4783-aca6-e7b3ab74513f kind: Event lastTimestamp: "2026-05-26T16:17:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:42Z" name: isvc-primary-ba2514-predictor-686dd7994-vcx9l.18b329d28c46b151 namespace: kserve-ci-e2e-test resourceVersion: "18011" uid: 69aba70d-5b58-4c64-b3f9-363e47aceea2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-ba2514-predictor-686dd7994-vcx9l namespace: kserve-ci-e2e-test resourceVersion: "17942" uid: 59ce4742-c702-4783-aca6-e7b3ab74513f kind: Event lastTimestamp: "2026-05-26T16:17:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:42Z" name: isvc-primary-ba2514-predictor-686dd7994-vcx9l.18b329d28cb54e05 namespace: kserve-ci-e2e-test resourceVersion: "18012" uid: 7aca5600-9188-4077-a76e-0da5230fbf1b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-ba2514-predictor-686dd7994-vcx9l namespace: kserve-ci-e2e-test resourceVersion: "17942" uid: 59ce4742-c702-4783-aca6-e7b3ab74513f kind: Event lastTimestamp: "2026-05-26T16:17:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:42Z" name: isvc-primary-ba2514-predictor-686dd7994-vcx9l.18b329d28cc085ba namespace: kserve-ci-e2e-test resourceVersion: "18013" uid: d7d5100e-9434-42c0-ba48-f7ef459b127e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-ba2514-predictor-686dd7994-vcx9l namespace: kserve-ci-e2e-test resourceVersion: "17942" uid: 59ce4742-c702-4783-aca6-e7b3ab74513f kind: Event lastTimestamp: "2026-05-26T16:17:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:42Z" name: isvc-primary-ba2514-predictor-686dd7994-vcx9l.18b329d28f99108b namespace: kserve-ci-e2e-test resourceVersion: "18016" uid: 182c8249-36d6-4d00-a8c7-ff902fc6db5e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-ba2514-predictor-686dd7994-vcx9l namespace: kserve-ci-e2e-test resourceVersion: "17942" uid: 59ce4742-c702-4783-aca6-e7b3ab74513f kind: Event lastTimestamp: "2026-05-26T16:17:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:42Z" name: isvc-primary-ba2514-predictor-686dd7994-vcx9l.18b329d290025b31 namespace: kserve-ci-e2e-test resourceVersion: "18017" uid: a2b528a7-677a-43d3-b965-2414ffb90c0b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-26T16:17:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-ba2514-predictor-686dd7994-vcx9l namespace: kserve-ci-e2e-test resourceVersion: "17942" uid: 59ce4742-c702-4783-aca6-e7b3ab74513f kind: Event lastTimestamp: "2026-05-26T16:18:38Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:18:38Z" name: isvc-primary-ba2514-predictor-686dd7994-vcx9l.18b329d2c5773c76 namespace: kserve-ci-e2e-test resourceVersion: "18430" uid: 0abe2739-4637-46f1-a9c9-4f4985755b60 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-ba2514-predictor-686dd7994-vcx9l namespace: kserve-ci-e2e-test resourceVersion: "17942" uid: 59ce4742-c702-4783-aca6-e7b3ab74513f kind: Event lastTimestamp: "2026-05-26T16:19:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:19:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:19:14Z" name: isvc-primary-ba2514-predictor-686dd7994-vcx9l.18b329e8292d82b8 namespace: kserve-ci-e2e-test resourceVersion: "18812" uid: b81b56c2-de9d-4ca7-86fc-f690e4733772 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-ba2514-predictor-686dd7994-vcx9l namespace: kserve-ci-e2e-test resourceVersion: "17942" uid: 59ce4742-c702-4783-aca6-e7b3ab74513f kind: Event lastTimestamp: "2026-05-26T16:19:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:19:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:19:14Z" name: isvc-primary-ba2514-predictor-686dd7994-vcx9l.18b329e8292e834c namespace: kserve-ci-e2e-test resourceVersion: "18814" uid: 0104e59c-b6cc-4b66-8d86-0cb99af76e18 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-ba2514-predictor-686dd7994 namespace: kserve-ci-e2e-test resourceVersion: "17928" uid: 9bbd982e-38b7-408c-a00f-ddeebdfea3a3 kind: Event lastTimestamp: "2026-05-26T16:17:36Z" message: 'Created pod: isvc-primary-ba2514-predictor-686dd7994-vcx9l' metadata: creationTimestamp: "2026-05-26T16:17:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:17:36Z" name: isvc-primary-ba2514-predictor-686dd7994.18b329d145b9a8da namespace: kserve-ci-e2e-test resourceVersion: "17940" uid: 2b725937-cec6-41db-8c12-29a2fb9aad96 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-ba2514-predictor namespace: kserve-ci-e2e-test resourceVersion: "17926" uid: 57d2836d-dc7d-4b71-a702-ed65127a45bd kind: Event lastTimestamp: "2026-05-26T16:17:36Z" message: Scaled up replica set isvc-primary-ba2514-predictor-686dd7994 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:17:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:17:36Z" name: isvc-primary-ba2514-predictor.18b329d1403899ac namespace: kserve-ci-e2e-test resourceVersion: "17931" uid: 700b3689-9acd-4334-85ec-b07c37dec9ea reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:17:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-ba2514-predictor namespace: kserve-ci-e2e-test resourceVersion: "17927" uid: d074dfdf-cd0a-474d-b132-fc07e4ad4774 kind: Event lastTimestamp: "2026-05-26T16:18:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:17:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:18:21Z" name: isvc-primary-ba2514-predictor.18b329d4bece6fa5 namespace: kserve-ci-e2e-test resourceVersion: "18305" uid: 9486100a-ece1-4710-a2ab-df8019c6e8a5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:17:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-ba2514-predictor namespace: kserve-ci-e2e-test resourceVersion: "17927" uid: d074dfdf-cd0a-474d-b132-fc07e4ad4774 kind: Event lastTimestamp: "2026-05-26T16:18:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:17:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:18:21Z" name: isvc-primary-ba2514-predictor.18b329d4bf599fa2 namespace: kserve-ci-e2e-test resourceVersion: "18306" uid: 0cf02a28-6e28-4d32-97ee-a7fd5ba6d3a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:18:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-ba2514-predictor namespace: kserve-ci-e2e-test resourceVersion: "18092" uid: d074dfdf-cd0a-474d-b132-fc07e4ad4774 kind: Event lastTimestamp: "2026-05-26T16:19:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:18:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:19:06Z" name: isvc-primary-ba2514-predictor.18b329df3ca96a04 namespace: kserve-ci-e2e-test resourceVersion: "18694" uid: 626ece16-9323-4db1-8494-45d5b885e931 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:18:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-ba2514-predictor namespace: kserve-ci-e2e-test resourceVersion: "18092" uid: d074dfdf-cd0a-474d-b132-fc07e4ad4774 kind: Event lastTimestamp: "2026-05-26T16:19:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:18:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:19:06Z" name: isvc-primary-ba2514-predictor.18b329df3d2957c9 namespace: kserve-ci-e2e-test resourceVersion: "18695" uid: f69080f3-f477-4a31-b2ab-300c809de080 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-05-26T16:17:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-ba2514 namespace: kserve-ci-e2e-test resourceVersion: "17900" uid: a2182775-f140-4e8b-8c09-c16206847949 kind: Event lastTimestamp: "2026-05-26T16:18:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:17:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:18:49Z" name: isvc-primary-ba2514.18b329d1400b2ee5 namespace: kserve-ci-e2e-test resourceVersion: "18505" uid: 9f09f8f4-08cb-4dbd-9278-03ff58420ba0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:17:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-ba2514 namespace: kserve-ci-e2e-test resourceVersion: "17900" uid: a2182775-f140-4e8b-8c09-c16206847949 kind: Event lastTimestamp: "2026-05-26T16:17:36Z" message: 'Failed to update status for InferenceService "isvc-primary-ba2514": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-ba2514": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:17:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:17:36Z" name: isvc-primary-ba2514.18b329d140a14b7a namespace: kserve-ci-e2e-test resourceVersion: "17951" uid: 50a9f8e3-6bed-473a-a5c8-d35c87d773ac reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:17:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-ba2514 namespace: kserve-ci-e2e-test resourceVersion: "17900" uid: a2182775-f140-4e8b-8c09-c16206847949 kind: Event lastTimestamp: "2026-05-26T16:17:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-ba2514": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:17:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:17:36Z" name: isvc-primary-ba2514.18b329d140a1b9ed namespace: kserve-ci-e2e-test resourceVersion: "17952" uid: 283e9921-637b-4cf5-b1f1-50a0fa7b2f52 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:18:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-ba2514 namespace: kserve-ci-e2e-test resourceVersion: "18502" uid: a2182775-f140-4e8b-8c09-c16206847949 kind: Event lastTimestamp: "2026-05-26T16:18:49Z" message: InferenceService [isvc-primary-ba2514] is Ready metadata: creationTimestamp: "2026-05-26T16:18:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:18:49Z" name: isvc-primary-ba2514.18b329e229aeaabe namespace: kserve-ci-e2e-test resourceVersion: "18504" uid: 07375ee2-bf26-4e44-b301-15bcc7d4f120 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:08:24.942193Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13024" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf to ip-10-0-134-19.ec2.internal metadata: creationTimestamp: "2026-05-26T16:08:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:08:24Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b32950d82994b4 namespace: kserve-ci-e2e-test resourceVersion: "13029" uid: 8482f5e1-483a-490a-bcf1-63aa067d7e95 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8b5dfd5d8-s8tjj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:25Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13037" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:08:25Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:08:25Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b32950f1cfdc4e namespace: kserve-ci-e2e-test resourceVersion: "13047" uid: 4ae3239c-5428-41ea-aaf9-f01dbabe5ca6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:08:25Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0413900e48a4ba69f75c45b06af88f792921f4669631dcf3a047c1fe5c1fc341" metadata: creationTimestamp: "2026-05-26T16:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:08:25Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b32950f38e822f namespace: kserve-ci-e2e-test resourceVersion: "13049" uid: 9025c570-ae03-4914-a0e8-b7bc4218008d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:08:29Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0413900e48a4ba69f75c45b06af88f792921f4669631dcf3a047c1fe5c1fc341" in 3.857s (3.857s including waiting). Image size: 301466548 bytes.' metadata: creationTimestamp: "2026-05-26T16:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:08:29Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b32951d982249a namespace: kserve-ci-e2e-test resourceVersion: "13085" uid: 760bede0-2887-4cd0-a501-b6336ad42589 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:08:29Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T16:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:08:29Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b32951dc794aab namespace: kserve-ci-e2e-test resourceVersion: "13086" uid: d2f3086d-5473-457d-a6fa-2efae01a6c81 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:08:29Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T16:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:08:29Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b32951dce6c69a namespace: kserve-ci-e2e-test resourceVersion: "13087" uid: 2babbe60-e4cc-4d57-9d7c-c14d96a62cc8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:08:33Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" metadata: creationTimestamp: "2026-05-26T16:08:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:08:33Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b32952c33882db namespace: kserve-ci-e2e-test resourceVersion: "13120" uid: 5660ad50-8019-49f0-90a4-adf2de05c69e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:08:46Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" in 13.094s (13.094s including waiting). Image size: 1560933300 bytes.' metadata: creationTimestamp: "2026-05-26T16:08:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:08:46Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b32955cfb0f97e namespace: kserve-ci-e2e-test resourceVersion: "13214" uid: be663b9c-4bfe-47f2-a2cf-837638bf681d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:08:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:08:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:08:46Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b32955d340d723 namespace: kserve-ci-e2e-test resourceVersion: "13215" uid: 76b180f0-05ad-40b6-a0bd-d3584f123538 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:08:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:08:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:08:46Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b32955d3ab09db namespace: kserve-ci-e2e-test resourceVersion: "13216" uid: 65428c90-349d-4b2d-85c4-a467b0c44695 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:08:46Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-26T16:08:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:08:46Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b32955d3b9a507 namespace: kserve-ci-e2e-test resourceVersion: "13217" uid: 12ed1e56-35b5-4e8c-a50b-444401c1a539 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:08:48Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.403s (2.403s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-26T16:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:08:48Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b329566301da56 namespace: kserve-ci-e2e-test resourceVersion: "13257" uid: 8e473d50-4d7d-4463-a005-f81a36a9a77c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:08:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:08:48Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b329566645a551 namespace: kserve-ci-e2e-test resourceVersion: "13259" uid: 4681c3c4-3da3-4b3a-b535-8fba286b78fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:08:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:08:48Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b3295666b4289f namespace: kserve-ci-e2e-test resourceVersion: "13260" uid: 9bb218ad-93dc-4816-a792-6c4ee0bff7f7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:08:48Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:d4dfc83be5dd58d8739a157278436e5a923b7626716bcc5e76e3c331a25671b6" metadata: creationTimestamp: "2026-05-26T16:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:08:48Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b3295666beed7a namespace: kserve-ci-e2e-test resourceVersion: "13262" uid: 06ebd463-163c-4d70-b026-a28683265e1e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:08:51Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:d4dfc83be5dd58d8739a157278436e5a923b7626716bcc5e76e3c331a25671b6" in 2.702s (2.702s including waiting). Image size: 239690326 bytes.' metadata: creationTimestamp: "2026-05-26T16:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:08:51Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b3295707d3e883 namespace: kserve-ci-e2e-test resourceVersion: "13293" uid: 745e8656-2b26-4732-bde5-7f3732175c22 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:08:51Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-05-26T16:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:08:51Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b329570acf9610 namespace: kserve-ci-e2e-test resourceVersion: "13294" uid: a7c4ab86-6e1e-4c7c-ac07-c94abd90c6f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:08:51Z" message: Started container agent metadata: creationTimestamp: "2026-05-26T16:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:08:51Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b329570b3b1e0d namespace: kserve-ci-e2e-test resourceVersion: "13295" uid: 7aa9f54e-2340-45e3-85f5-6764e789b013 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-05-26T16:08:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:10:25Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:25Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b329573487c4c5 namespace: kserve-ci-e2e-test resourceVersion: "14154" uid: 6112aa7b-4c0e-4fe3-98a5-915dd2a5791f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-26T16:08:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:10:25Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T16:08:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:25Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b3295770506214 namespace: kserve-ci-e2e-test resourceVersion: "14155" uid: 6ac6ef8d-4cc7-4c55-99eb-0ad27b050f12 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:10:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:10Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b329694e53356e namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: 0f057d70-3376-4a86-80e9-290bd349b755 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:10:10Z" message: Stopping container agent metadata: creationTimestamp: "2026-05-26T16:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:10Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b329694e55a97b namespace: kserve-ci-e2e-test resourceVersion: "13928" uid: 5642ae0d-bb64-4c1f-baca-63d5c741a105 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:10:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:10Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b329694e5651e1 namespace: kserve-ci-e2e-test resourceVersion: "13931" uid: e45dfba1-3ab9-4f2f-b7cf-7709d787c522 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T16:10:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 151ad032-daaf-4774-b54b-c4f98d55891d kind: Event lastTimestamp: "2026-05-26T16:10:28Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:10:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:28Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf.18b3296a10ac2d11 namespace: kserve-ci-e2e-test resourceVersion: "14176" uid: 4003e63d-047a-4755-b551-6e23a2550aa9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7 namespace: kserve-ci-e2e-test resourceVersion: "13016" uid: b6b7acda-e936-4f1c-b303-6e8b608c6e4c kind: Event lastTimestamp: "2026-05-26T16:08:24Z" message: 'Created pod: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7-bv7nf' metadata: creationTimestamp: "2026-05-26T16:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:08:24Z" name: isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7.18b32950d7907c41 namespace: kserve-ci-e2e-test resourceVersion: "13025" uid: c3ad681b-bce5-441e-b79d-864c202919fc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-91b4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "13015" uid: caa069c8-e9e2-4da3-8fc7-b7171bf90a1f kind: Event lastTimestamp: "2026-05-26T16:08:24Z" message: Scaled up replica set isvc-raw-sklearn-batcher-91b4a-predictor-78f8d8cdc7 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:08:24Z" name: isvc-raw-sklearn-batcher-91b4a-predictor.18b32950d23e53b8 namespace: kserve-ci-e2e-test resourceVersion: "13017" uid: db3538a5-527e-4725-b1ad-f9d0bf0c36d9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:08:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-91b4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "13028" uid: ea2c6392-a54b-4b64-98c8-8245e753add5 kind: Event lastTimestamp: "2026-05-26T16:09:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:09:10Z" name: isvc-raw-sklearn-batcher-91b4a-predictor.18b3295458063327 namespace: kserve-ci-e2e-test resourceVersion: "13462" uid: ebb9e079-f0ae-49a3-92d5-ef0ed40e4de3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:08:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-91b4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "13028" uid: ea2c6392-a54b-4b64-98c8-8245e753add5 kind: Event lastTimestamp: "2026-05-26T16:09:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:09:10Z" name: isvc-raw-sklearn-batcher-91b4a-predictor.18b329545889423d namespace: kserve-ci-e2e-test resourceVersion: "13463" uid: 1ab752cc-f382-419b-ac32-f256eddd0106 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:09:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-91b4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "13173" uid: ea2c6392-a54b-4b64-98c8-8245e753add5 kind: Event lastTimestamp: "2026-05-26T16:09:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:09:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:09:55Z" name: isvc-raw-sklearn-batcher-91b4a-predictor.18b3295ed5f81ace namespace: kserve-ci-e2e-test resourceVersion: "13782" uid: 020b163f-9436-4b8f-bb0b-267096d71e11 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:09:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-91b4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "13173" uid: ea2c6392-a54b-4b64-98c8-8245e753add5 kind: Event lastTimestamp: "2026-05-26T16:09:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:09:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:09:55Z" name: isvc-raw-sklearn-batcher-91b4a-predictor.18b3295ed6712860 namespace: kserve-ci-e2e-test resourceVersion: "13783" uid: 3f5fb07c-d87b-42bb-8cf0-ca05e3f71820 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-05-26T16:08:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-91b4a namespace: kserve-ci-e2e-test resourceVersion: "13005" uid: 7d7f5aac-4aa8-4861-aa09-f92687d73221 kind: Event lastTimestamp: "2026-05-26T16:09:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:09:55Z" name: isvc-raw-sklearn-batcher-91b4a.18b32950d8442581 namespace: kserve-ci-e2e-test resourceVersion: "13798" uid: 0bc1daa7-ffc6-4d70-b204-7432a1af7662 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-91b4a namespace: kserve-ci-e2e-test resourceVersion: "13005" uid: 7d7f5aac-4aa8-4861-aa09-f92687d73221 kind: Event lastTimestamp: "2026-05-26T16:08:24Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-91b4a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-91b4a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:08:24Z" name: isvc-raw-sklearn-batcher-91b4a.18b32950d8bdb06d namespace: kserve-ci-e2e-test resourceVersion: "13032" uid: 69b9c893-906e-4032-8dfa-aab6e47e7c22 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:08:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-91b4a namespace: kserve-ci-e2e-test resourceVersion: "13005" uid: 7d7f5aac-4aa8-4861-aa09-f92687d73221 kind: Event lastTimestamp: "2026-05-26T16:08:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-91b4a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:08:24Z" name: isvc-raw-sklearn-batcher-91b4a.18b32950d8bdf557 namespace: kserve-ci-e2e-test resourceVersion: "13034" uid: c922c2ee-dc00-418d-9dd9-c3d87e37b48f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:09:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-91b4a namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: 7d7f5aac-4aa8-4861-aa09-f92687d73221 kind: Event lastTimestamp: "2026-05-26T16:09:55Z" message: InferenceService [isvc-raw-sklearn-batcher-91b4a] is Ready metadata: creationTimestamp: "2026-05-26T16:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:09:55Z" name: isvc-raw-sklearn-batcher-91b4a.18b32965e894f46c namespace: kserve-ci-e2e-test resourceVersion: "13796" uid: 3fbc1194-e669-43bf-aa7c-a4748a4c3a7c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:18:56.695625Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-ba2514-predictor-6b8dc68545-bl4zz namespace: kserve-ci-e2e-test resourceVersion: "18591" uid: c8fe44d0-deee-4c97-943f-a7a0a51c500a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-ba2514-predictor-6b8dc68545-bl4zz to ip-10-0-134-19.ec2.internal metadata: creationTimestamp: "2026-05-26T16:18:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:18:56Z" name: isvc-secondary-ba2514-predictor-6b8dc68545-bl4zz.18b329e3ef99e259 namespace: kserve-ci-e2e-test resourceVersion: "18597" uid: 88894dbf-1456-4d07-bc88-d6423e9b5757 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8b5dfd5d8-s8tjj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:18:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-ba2514-predictor-6b8dc68545-bl4zz namespace: kserve-ci-e2e-test resourceVersion: "18601" uid: c8fe44d0-deee-4c97-943f-a7a0a51c500a kind: Event lastTimestamp: "2026-05-26T16:18:57Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:18:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:18:57Z" name: isvc-secondary-ba2514-predictor-6b8dc68545-bl4zz.18b329e408cd6baf namespace: kserve-ci-e2e-test resourceVersion: "18610" uid: 5dd25dd5-b20d-48c1-b7aa-e540f0b16fec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:18:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-ba2514-predictor-6b8dc68545-bl4zz namespace: kserve-ci-e2e-test resourceVersion: "18595" uid: c8fe44d0-deee-4c97-943f-a7a0a51c500a kind: Event lastTimestamp: "2026-05-26T16:19:03Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0413900e48a4ba69f75c45b06af88f792921f4669631dcf3a047c1fe5c1fc341" already present on machine metadata: creationTimestamp: "2026-05-26T16:18:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:19:03Z" name: isvc-secondary-ba2514-predictor-6b8dc68545-bl4zz.18b329e40a686a1e namespace: kserve-ci-e2e-test resourceVersion: "18663" uid: c348a675-3f10-4117-8242-62576736dde4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:18:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-ba2514-predictor-6b8dc68545-bl4zz namespace: kserve-ci-e2e-test resourceVersion: "18595" uid: c8fe44d0-deee-4c97-943f-a7a0a51c500a kind: Event lastTimestamp: "2026-05-26T16:19:03Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T16:18:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:19:03Z" name: isvc-secondary-ba2514-predictor-6b8dc68545-bl4zz.18b329e40e1f81f3 namespace: kserve-ci-e2e-test resourceVersion: "18665" uid: 7572b13f-d47b-42d2-b2b4-70413506e8b9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:18:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-ba2514-predictor-6b8dc68545-bl4zz namespace: kserve-ci-e2e-test resourceVersion: "18595" uid: c8fe44d0-deee-4c97-943f-a7a0a51c500a kind: Event lastTimestamp: "2026-05-26T16:19:03Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T16:18:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:19:03Z" name: isvc-secondary-ba2514-predictor-6b8dc68545-bl4zz.18b329e40e905ea6 namespace: kserve-ci-e2e-test resourceVersion: "18666" uid: d0314896-4ff0-4e07-bc9c-0642324b6f10 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-ba2514-predictor-6b8dc68545-bl4zz namespace: kserve-ci-e2e-test resourceVersion: "18595" uid: c8fe44d0-deee-4c97-943f-a7a0a51c500a kind: Event lastTimestamp: "2026-05-26T16:19:08Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-ba2514-predictor-6b8dc68545-bl4zz_kserve-ci-e2e-test(c8fe44d0-deee-4c97-943f-a7a0a51c500a) metadata: creationTimestamp: "2026-05-26T16:19:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:19:08Z" name: isvc-secondary-ba2514-predictor-6b8dc68545-bl4zz.18b329e6a03a60e0 namespace: kserve-ci-e2e-test resourceVersion: "18706" uid: a1a830a6-a2f6-4761-8016-01b4958b8d76 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:18:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-ba2514-predictor-6b8dc68545 namespace: kserve-ci-e2e-test resourceVersion: "18582" uid: 1ca1b845-40c9-490d-b910-d0b538b33e77 kind: Event lastTimestamp: "2026-05-26T16:18:56Z" message: 'Created pod: isvc-secondary-ba2514-predictor-6b8dc68545-bl4zz' metadata: creationTimestamp: "2026-05-26T16:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:18:56Z" name: isvc-secondary-ba2514-predictor-6b8dc68545.18b329e3eee440f5 namespace: kserve-ci-e2e-test resourceVersion: "18594" uid: d58ede9a-c718-47cf-ac01-9a2f2e602f6d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:18:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-ba2514-predictor namespace: kserve-ci-e2e-test resourceVersion: "18581" uid: c642cec8-043b-48ff-a6ab-37c4095b6649 kind: Event lastTimestamp: "2026-05-26T16:18:56Z" message: Scaled up replica set isvc-secondary-ba2514-predictor-6b8dc68545 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:18:56Z" name: isvc-secondary-ba2514-predictor.18b329e3eaba051c namespace: kserve-ci-e2e-test resourceVersion: "18585" uid: c930ccc2-bd79-49fd-86e5-f73207fa511c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-ba2514-predictor namespace: kserve-ci-e2e-test resourceVersion: "18583" uid: e6f11743-b25c-4029-949d-bca95347fbac kind: Event lastTimestamp: "2026-05-26T16:19:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:19:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:19:11Z" name: isvc-secondary-ba2514-predictor.18b329e769f9ac50 namespace: kserve-ci-e2e-test resourceVersion: "18732" uid: 5b49925d-a4db-43d4-b425-3feed7c82208 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-ba2514-predictor namespace: kserve-ci-e2e-test resourceVersion: "18583" uid: e6f11743-b25c-4029-949d-bca95347fbac kind: Event lastTimestamp: "2026-05-26T16:19:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:19:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:19:11Z" name: isvc-secondary-ba2514-predictor.18b329e76a91862e namespace: kserve-ci-e2e-test resourceVersion: "18734" uid: a909db07-9a73-4c13-9baa-afde0bcaa78f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-05-26T16:18:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-ba2514 namespace: kserve-ci-e2e-test resourceVersion: "18572" uid: a6d74b04-1d18-4782-85d1-492ba46873e6 kind: Event lastTimestamp: "2026-05-26T16:19:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:19:09Z" name: isvc-secondary-ba2514.18b329e3eb3ba20d namespace: kserve-ci-e2e-test resourceVersion: "18721" uid: 2788ca31-3ac4-409f-b0b0-18c350039f5b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:18:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-ba2514 namespace: kserve-ci-e2e-test resourceVersion: "18572" uid: a6d74b04-1d18-4782-85d1-492ba46873e6 kind: Event lastTimestamp: "2026-05-26T16:18:56Z" message: 'Failed to update status for InferenceService "isvc-secondary-ba2514": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-ba2514": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:18:56Z" name: isvc-secondary-ba2514.18b329e3ebf70524 namespace: kserve-ci-e2e-test resourceVersion: "18588" uid: 93a45f1f-9be2-4cf6-8221-cea76d610650 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:18:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-ba2514 namespace: kserve-ci-e2e-test resourceVersion: "18572" uid: a6d74b04-1d18-4782-85d1-492ba46873e6 kind: Event lastTimestamp: "2026-05-26T16:18:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-ba2514": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:18:56Z" name: isvc-secondary-ba2514.18b329e3ebf74e36 namespace: kserve-ci-e2e-test resourceVersion: "18590" uid: caf79788-c415-408c-9b91-d645e777a36a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:10:10.155739Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8 namespace: kserve-ci-e2e-test resourceVersion: "13941" uid: c1dbdd50-e6c5-4ac1-b33e-56f276cac3f6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8 to ip-10-0-134-19.ec2.internal metadata: creationTimestamp: "2026-05-26T16:10:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:10:10Z" name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8.18b329695760d3e5 namespace: kserve-ci-e2e-test resourceVersion: "13951" uid: d372f1d9-be22-41ba-9f2d-ac138f1f5150 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8b5dfd5d8-s8tjj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8 namespace: kserve-ci-e2e-test resourceVersion: "13946" uid: c1dbdd50-e6c5-4ac1-b33e-56f276cac3f6 kind: Event lastTimestamp: "2026-05-26T16:10:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-5c32b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:10Z" name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8.18b329696151526a namespace: kserve-ci-e2e-test resourceVersion: "13972" uid: f0186f9b-2ecd-45bf-bc8a-8d50cf63bc33 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8 namespace: kserve-ci-e2e-test resourceVersion: "13961" uid: c1dbdd50-e6c5-4ac1-b33e-56f276cac3f6 kind: Event lastTimestamp: "2026-05-26T16:10:11Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:10:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:10:11Z" name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8.18b32969947ca627 namespace: kserve-ci-e2e-test resourceVersion: "14003" uid: 957e3f1a-f3d9-4fc3-b07a-7fc148914f7c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8 namespace: kserve-ci-e2e-test resourceVersion: "13946" uid: c1dbdd50-e6c5-4ac1-b33e-56f276cac3f6 kind: Event lastTimestamp: "2026-05-26T16:10:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0413900e48a4ba69f75c45b06af88f792921f4669631dcf3a047c1fe5c1fc341" already present on machine metadata: creationTimestamp: "2026-05-26T16:10:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:11Z" name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8.18b329699626b1ef namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 0bc83fb3-feb6-4658-9cd9-71c5b21c00cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8 namespace: kserve-ci-e2e-test resourceVersion: "13946" uid: c1dbdd50-e6c5-4ac1-b33e-56f276cac3f6 kind: Event lastTimestamp: "2026-05-26T16:10:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T16:10:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:11Z" name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8.18b3296998e1bfd0 namespace: kserve-ci-e2e-test resourceVersion: "14006" uid: 9bf7b90d-c593-4495-beff-0538be79765d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8 namespace: kserve-ci-e2e-test resourceVersion: "13946" uid: c1dbdd50-e6c5-4ac1-b33e-56f276cac3f6 kind: Event lastTimestamp: "2026-05-26T16:10:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T16:10:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:11Z" name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8.18b32969994df6df namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: 27cfdf3d-f849-47a0-ba18-c135e0b8b1cd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8 namespace: kserve-ci-e2e-test resourceVersion: "13946" uid: c1dbdd50-e6c5-4ac1-b33e-56f276cac3f6 kind: Event lastTimestamp: "2026-05-26T16:10:15Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:10:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:15Z" name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8.18b3296a97fd54f8 namespace: kserve-ci-e2e-test resourceVersion: "14061" uid: 62e46339-3a66-4acc-8427-3afdc4e4020f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8 namespace: kserve-ci-e2e-test resourceVersion: "13946" uid: c1dbdd50-e6c5-4ac1-b33e-56f276cac3f6 kind: Event lastTimestamp: "2026-05-26T16:10:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:10:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:15Z" name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8.18b3296a9ac20c19 namespace: kserve-ci-e2e-test resourceVersion: "14063" uid: ae24f1ef-3277-4ece-87bd-1acda349a1b2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8 namespace: kserve-ci-e2e-test resourceVersion: "13946" uid: c1dbdd50-e6c5-4ac1-b33e-56f276cac3f6 kind: Event lastTimestamp: "2026-05-26T16:10:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:10:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:15Z" name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8.18b3296a9b2fc5ac namespace: kserve-ci-e2e-test resourceVersion: "14064" uid: 278c69c7-b144-4c90-aa73-69150f3d7b2d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8 namespace: kserve-ci-e2e-test resourceVersion: "13946" uid: c1dbdd50-e6c5-4ac1-b33e-56f276cac3f6 kind: Event lastTimestamp: "2026-05-26T16:10:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:10:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:15Z" name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8.18b3296a9b3a4997 namespace: kserve-ci-e2e-test resourceVersion: "14065" uid: 2ed00de6-b74f-4880-84bb-386d8e8ee07b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8 namespace: kserve-ci-e2e-test resourceVersion: "13946" uid: c1dbdd50-e6c5-4ac1-b33e-56f276cac3f6 kind: Event lastTimestamp: "2026-05-26T16:10:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:10:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:15Z" name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8.18b3296a9e6ecf3a namespace: kserve-ci-e2e-test resourceVersion: "14069" uid: 1760b8ce-235f-4f87-aed7-c2acf23aad77 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8 namespace: kserve-ci-e2e-test resourceVersion: "13946" uid: c1dbdd50-e6c5-4ac1-b33e-56f276cac3f6 kind: Event lastTimestamp: "2026-05-26T16:10:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:10:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:15Z" name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8.18b3296a9ed5c4ca namespace: kserve-ci-e2e-test resourceVersion: "14070" uid: ea2f48c2-cf75-4f90-b33f-84284449ade5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T16:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8 namespace: kserve-ci-e2e-test resourceVersion: "13946" uid: c1dbdd50-e6c5-4ac1-b33e-56f276cac3f6 kind: Event lastTimestamp: "2026-05-26T16:11:52Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:52Z" name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8.18b3296ad3e88480 namespace: kserve-ci-e2e-test resourceVersion: "15006" uid: 5f6aee34-37cd-4f4c-b749-d3378decf64f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8 namespace: kserve-ci-e2e-test resourceVersion: "13946" uid: c1dbdd50-e6c5-4ac1-b33e-56f276cac3f6 kind: Event lastTimestamp: "2026-05-26T16:11:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:50Z" name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8.18b32980b6dfc5df namespace: kserve-ci-e2e-test resourceVersion: "14920" uid: 3beb8e8f-76c5-4135-a912-23aba8b6a5d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8 namespace: kserve-ci-e2e-test resourceVersion: "13946" uid: c1dbdd50-e6c5-4ac1-b33e-56f276cac3f6 kind: Event lastTimestamp: "2026-05-26T16:11:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:50Z" name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8.18b32980b6e07af1 namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 860f1ccd-ea76-4b83-b229-0d2037a17808 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8 namespace: kserve-ci-e2e-test resourceVersion: "13946" uid: c1dbdd50-e6c5-4ac1-b33e-56f276cac3f6 kind: Event lastTimestamp: "2026-05-26T16:11:52Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:52Z" name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8.18b329812e1c4e32 namespace: kserve-ci-e2e-test resourceVersion: "15005" uid: d65b30f8-fd29-4031-bdaa-b196c1dfc3c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f namespace: kserve-ci-e2e-test resourceVersion: "13927" uid: a49583f0-b831-4f69-8216-5c88f78cf206 kind: Event lastTimestamp: "2026-05-26T16:10:10Z" message: 'Created pod: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f-7mvh8' metadata: creationTimestamp: "2026-05-26T16:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:10:10Z" name: isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f.18b3296955e8fe03 namespace: kserve-ci-e2e-test resourceVersion: "13944" uid: 3b517f30-4952-4146-8d33-cdd269568b2b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-5c32b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13924" uid: 6ed5e3ab-36a1-4ca1-9c89-e749d6033dd6 kind: Event lastTimestamp: "2026-05-26T16:10:10Z" message: Scaled up replica set isvc-sklearn-graph-raw-5c32b-predictor-86f64c4b8f from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:10:10Z" name: isvc-sklearn-graph-raw-5c32b-predictor.18b329694f1dbb17 namespace: kserve-ci-e2e-test resourceVersion: "13930" uid: 56e54597-d362-4e43-8403-1a52963c9123 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:10:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-5c32b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13926" uid: da7e10f6-e2a0-47c1-9a4b-5c0e4759e8b7 kind: Event lastTimestamp: "2026-05-26T16:10:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:10:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:10:40Z" name: isvc-sklearn-graph-raw-5c32b-predictor.18b3296ccdd3f68a namespace: kserve-ci-e2e-test resourceVersion: "14265" uid: 91d672ae-1555-43db-8c30-6661244ccc61 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:10:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-5c32b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13926" uid: da7e10f6-e2a0-47c1-9a4b-5c0e4759e8b7 kind: Event lastTimestamp: "2026-05-26T16:10:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:10:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:10:40Z" name: isvc-sklearn-graph-raw-5c32b-predictor.18b3296cce4e46c6 namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 9b5e69c4-3dc8-4324-b83c-3dcb92fc4d0b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T16:10:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-5c32b-predictor namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: da7e10f6-e2a0-47c1-9a4b-5c0e4759e8b7 kind: Event lastTimestamp: "2026-05-26T16:11:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:10:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:11:40Z" name: isvc-sklearn-graph-raw-5c32b-predictor.18b32973ccbf57fa namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: 7517fe6c-9d4b-4268-9a27-a505559ecfe3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T16:10:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-5c32b-predictor namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: da7e10f6-e2a0-47c1-9a4b-5c0e4759e8b7 kind: Event lastTimestamp: "2026-05-26T16:11:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:10:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:11:40Z" name: isvc-sklearn-graph-raw-5c32b-predictor.18b32973cd8df2cc namespace: kserve-ci-e2e-test resourceVersion: "14745" uid: 12f0c9d9-f0bd-4fbd-8020-f199eb05ac20 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T16:10:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-5c32b namespace: kserve-ci-e2e-test resourceVersion: "13898" uid: c2feb0d2-7b30-44e7-8142-cfe1b53e2f24 kind: Event lastTimestamp: "2026-05-26T16:11:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:11:22Z" name: isvc-sklearn-graph-raw-5c32b.18b329694ef1872b namespace: kserve-ci-e2e-test resourceVersion: "14595" uid: 4e96fe4d-37df-448a-89e7-be00cb684a58 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-5c32b namespace: kserve-ci-e2e-test resourceVersion: "13898" uid: c2feb0d2-7b30-44e7-8142-cfe1b53e2f24 kind: Event lastTimestamp: "2026-05-26T16:10:10Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-5c32b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-5c32b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:10:10Z" name: isvc-sklearn-graph-raw-5c32b.18b329694f61cabc namespace: kserve-ci-e2e-test resourceVersion: "13933" uid: c51ae174-ca8e-4640-9325-b1a9a6c2c522 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-5c32b namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: c2feb0d2-7b30-44e7-8142-cfe1b53e2f24 kind: Event lastTimestamp: "2026-05-26T16:11:22Z" message: InferenceService [isvc-sklearn-graph-raw-5c32b] is Ready metadata: creationTimestamp: "2026-05-26T16:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:11:22Z" name: isvc-sklearn-graph-raw-5c32b.18b3297a38e52dfe namespace: kserve-ci-e2e-test resourceVersion: "14594" uid: 3081025b-3aa4-4326-9bb4-6142ccedd358 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:11:50.586554Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd namespace: kserve-ci-e2e-test resourceVersion: "14922" uid: 43de9a6e-da72-41b0-97ca-0e4a7dadb2f8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd to ip-10-0-134-19.ec2.internal metadata: creationTimestamp: "2026-05-26T16:11:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:11:50Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd.18b32980b9857512 namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: d62e4cb6-04d4-4518-bb9c-6032e40d7b92 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8b5dfd5d8-s8tjj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd namespace: kserve-ci-e2e-test resourceVersion: "14945" uid: 43de9a6e-da72-41b0-97ca-0e4a7dadb2f8 kind: Event lastTimestamp: "2026-05-26T16:11:51Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:11:51Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd.18b32980d2f1af86 namespace: kserve-ci-e2e-test resourceVersion: "14973" uid: ea888a77-6b9d-43ef-9346-51dfc19c65e1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 43de9a6e-da72-41b0-97ca-0e4a7dadb2f8 kind: Event lastTimestamp: "2026-05-26T16:11:51Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0413900e48a4ba69f75c45b06af88f792921f4669631dcf3a047c1fe5c1fc341" already present on machine metadata: creationTimestamp: "2026-05-26T16:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:51Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd.18b32980d4b40275 namespace: kserve-ci-e2e-test resourceVersion: "14975" uid: 1cb8be2e-41a7-47f0-b5e5-28b0be4c8570 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 43de9a6e-da72-41b0-97ca-0e4a7dadb2f8 kind: Event lastTimestamp: "2026-05-26T16:11:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T16:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:51Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd.18b32980d6f898c8 namespace: kserve-ci-e2e-test resourceVersion: "14976" uid: 85fb887a-9d3b-4206-98c2-5e22f344a050 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 43de9a6e-da72-41b0-97ca-0e4a7dadb2f8 kind: Event lastTimestamp: "2026-05-26T16:11:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T16:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:51Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd.18b32980d755c74b namespace: kserve-ci-e2e-test resourceVersion: "14977" uid: f846ff02-a2f2-42d5-8a3c-ac2213f8e020 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 43de9a6e-da72-41b0-97ca-0e4a7dadb2f8 kind: Event lastTimestamp: "2026-05-26T16:11:54Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:54Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd.18b32981b7996452 namespace: kserve-ci-e2e-test resourceVersion: "15032" uid: 98663f86-6769-458e-bbd9-87f061f735d2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 43de9a6e-da72-41b0-97ca-0e4a7dadb2f8 kind: Event lastTimestamp: "2026-05-26T16:11:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:54Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd.18b32981badb13ce namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 55488a84-9eef-4240-a40e-22a3f248a1a5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 43de9a6e-da72-41b0-97ca-0e4a7dadb2f8 kind: Event lastTimestamp: "2026-05-26T16:11:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:54Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd.18b32981bb3b5e84 namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: cb6f4e47-cacd-4a48-9182-6df5e98e008b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 43de9a6e-da72-41b0-97ca-0e4a7dadb2f8 kind: Event lastTimestamp: "2026-05-26T16:11:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:54Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd.18b32981bb4704ce namespace: kserve-ci-e2e-test resourceVersion: "15038" uid: 4bdd7a76-b415-4ea3-a373-3ee11882298e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 43de9a6e-da72-41b0-97ca-0e4a7dadb2f8 kind: Event lastTimestamp: "2026-05-26T16:11:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:54Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd.18b32981bd8272b4 namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: 4b7c89da-8ae1-4e6e-87b1-688db5d22040 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 43de9a6e-da72-41b0-97ca-0e4a7dadb2f8 kind: Event lastTimestamp: "2026-05-26T16:11:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:54Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd.18b32981bde03c04 namespace: kserve-ci-e2e-test resourceVersion: "15043" uid: 41ed074e-c830-435f-b0f1-5d79bcc0e665 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T16:11:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 43de9a6e-da72-41b0-97ca-0e4a7dadb2f8 kind: Event lastTimestamp: "2026-05-26T16:13:31Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:31Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd.18b32981f38c9b85 namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: 065d6395-3bc9-4a3f-8c00-269a637121df reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 43de9a6e-da72-41b0-97ca-0e4a7dadb2f8 kind: Event lastTimestamp: "2026-05-26T16:13:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:30Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd.18b329980e07a26f namespace: kserve-ci-e2e-test resourceVersion: "15868" uid: 4ac816f5-8759-4d1a-ba5e-531a74c01766 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 43de9a6e-da72-41b0-97ca-0e4a7dadb2f8 kind: Event lastTimestamp: "2026-05-26T16:13:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:30Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd.18b329980e07f650 namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: 82bb04fd-2520-4e1c-bfba-0e0568dc3121 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 43de9a6e-da72-41b0-97ca-0e4a7dadb2f8 kind: Event lastTimestamp: "2026-05-26T16:13:31Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp 10.133.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:13:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:31Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd.18b329984dc01607 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: cba9ba61-0f75-47bb-accd-5e6f137cd9cf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc namespace: kserve-ci-e2e-test resourceVersion: "14897" uid: 604e490a-9d61-42bc-a6bc-8ad94f447e71 kind: Event lastTimestamp: "2026-05-26T16:11:50Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc-ct4hd' metadata: creationTimestamp: "2026-05-26T16:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:11:50Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc.18b32980b86766fa namespace: kserve-ci-e2e-test resourceVersion: "14925" uid: 73a6386b-a33c-4268-85f9-3fce6593788d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-89f63-predictor namespace: kserve-ci-e2e-test resourceVersion: "14891" uid: a0dcbc02-6597-47eb-9c60-18582036ebee kind: Event lastTimestamp: "2026-05-26T16:11:50Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-89f63-predictor-564cbb96cc from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:11:50Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor.18b32980b0f6d4fc namespace: kserve-ci-e2e-test resourceVersion: "14900" uid: fd26b652-1286-4c04-b447-134b81fcf6b4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:12:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-89f63-predictor namespace: kserve-ci-e2e-test resourceVersion: "14898" uid: dcd91224-5715-4f59-ab6e-7eda7a80b8bc kind: Event lastTimestamp: "2026-05-26T16:12:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:12:35Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor.18b32984305d1688 namespace: kserve-ci-e2e-test resourceVersion: "15371" uid: 2586adff-9393-416d-b2e4-3c065cd8ca62 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:12:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-89f63-predictor namespace: kserve-ci-e2e-test resourceVersion: "14898" uid: dcd91224-5715-4f59-ab6e-7eda7a80b8bc kind: Event lastTimestamp: "2026-05-26T16:12:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:12:35Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor.18b32984313a0b82 namespace: kserve-ci-e2e-test resourceVersion: "15372" uid: 1a8ead0a-4a43-4da9-a13d-b9900ce6e457 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:12:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-89f63-predictor namespace: kserve-ci-e2e-test resourceVersion: "15146" uid: dcd91224-5715-4f59-ab6e-7eda7a80b8bc kind: Event lastTimestamp: "2026-05-26T16:13:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:12:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:13:20Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor.18b3298eaf3308d3 namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 8f7d036f-d5a1-4eeb-a843-dbd04e484e58 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:12:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-89f63-predictor namespace: kserve-ci-e2e-test resourceVersion: "15146" uid: dcd91224-5715-4f59-ab6e-7eda7a80b8bc kind: Event lastTimestamp: "2026-05-26T16:13:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:12:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:13:20Z" name: isvc-sklearn-graph-raw-hpa-89f63-predictor.18b3298eafe99acd namespace: kserve-ci-e2e-test resourceVersion: "15738" uid: 7af70200-3076-4c61-a5c6-4d2abff4426b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T16:11:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-89f63 namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 5708b604-f28f-4350-97d3-a0a9ea8ef343 kind: Event lastTimestamp: "2026-05-26T16:13:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:13:02Z" name: isvc-sklearn-graph-raw-hpa-89f63.18b32980b149e897 namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: 53d54970-d962-476a-84e6-c54c373b3a41 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-89f63 namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 5708b604-f28f-4350-97d3-a0a9ea8ef343 kind: Event lastTimestamp: "2026-05-26T16:11:50Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-89f63": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-89f63": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:11:50Z" name: isvc-sklearn-graph-raw-hpa-89f63.18b32980b260d189 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: 5bce9a6d-11ac-4230-a70d-87c3ef933ab1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-89f63 namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: 5708b604-f28f-4350-97d3-a0a9ea8ef343 kind: Event lastTimestamp: "2026-05-26T16:13:01Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-89f63] is Ready metadata: creationTimestamp: "2026-05-26T16:13:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:13:01Z" name: isvc-sklearn-graph-raw-hpa-89f63.18b3299158460e30 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: 4ebdc4b0-4936-42f6-8342-31475ac7c47f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:15:16.215695Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n namespace: kserve-ci-e2e-test resourceVersion: "16821" uid: 55cb3473-0b1e-498a-bbcd-70b6660e763b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n to ip-10-0-134-19.ec2.internal metadata: creationTimestamp: "2026-05-26T16:15:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:15:16Z" name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n.18b329b099f91b5b namespace: kserve-ci-e2e-test resourceVersion: "16830" uid: c2c42506-0722-4b7a-8caf-c2b56d39b556 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8b5dfd5d8-s8tjj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n namespace: kserve-ci-e2e-test resourceVersion: "16846" uid: 55cb3473-0b1e-498a-bbcd-70b6660e763b kind: Event lastTimestamp: "2026-05-26T16:15:16Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:15:16Z" name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n.18b329b0b31e1905 namespace: kserve-ci-e2e-test resourceVersion: "16859" uid: 0b0bec38-e044-4834-bbcc-2ca6ee570592 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 55cb3473-0b1e-498a-bbcd-70b6660e763b kind: Event lastTimestamp: "2026-05-26T16:15:16Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0413900e48a4ba69f75c45b06af88f792921f4669631dcf3a047c1fe5c1fc341" already present on machine metadata: creationTimestamp: "2026-05-26T16:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:16Z" name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n.18b329b0b4b61689 namespace: kserve-ci-e2e-test resourceVersion: "16861" uid: 99ef20c1-9ef4-48c8-a24f-e776759e2bac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 55cb3473-0b1e-498a-bbcd-70b6660e763b kind: Event lastTimestamp: "2026-05-26T16:15:16Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T16:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:16Z" name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n.18b329b0b77d4ed5 namespace: kserve-ci-e2e-test resourceVersion: "16864" uid: c1169e47-bfb1-4940-97d9-4550c577be74 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 55cb3473-0b1e-498a-bbcd-70b6660e763b kind: Event lastTimestamp: "2026-05-26T16:15:16Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T16:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:16Z" name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n.18b329b0b7ed960c namespace: kserve-ci-e2e-test resourceVersion: "16865" uid: e960d968-ed00-416a-9082-28ba2635cde5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 55cb3473-0b1e-498a-bbcd-70b6660e763b kind: Event lastTimestamp: "2026-05-26T16:15:20Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:15:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:20Z" name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n.18b329b19a6d60b0 namespace: kserve-ci-e2e-test resourceVersion: "16901" uid: a49223ef-9112-441e-b9d8-63dfc39b7276 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 55cb3473-0b1e-498a-bbcd-70b6660e763b kind: Event lastTimestamp: "2026-05-26T16:15:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:15:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:20Z" name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n.18b329b19d8ffef1 namespace: kserve-ci-e2e-test resourceVersion: "16903" uid: f7b33b37-a11a-4afb-889a-09558047b093 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 55cb3473-0b1e-498a-bbcd-70b6660e763b kind: Event lastTimestamp: "2026-05-26T16:15:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:15:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:20Z" name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n.18b329b19e0999dc namespace: kserve-ci-e2e-test resourceVersion: "16904" uid: 2664fd0c-0f3e-4526-992c-b279621db917 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 55cb3473-0b1e-498a-bbcd-70b6660e763b kind: Event lastTimestamp: "2026-05-26T16:15:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:15:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:20Z" name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n.18b329b19e15180f namespace: kserve-ci-e2e-test resourceVersion: "16905" uid: f93172d1-d071-4022-9535-6e5d24594f8d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 55cb3473-0b1e-498a-bbcd-70b6660e763b kind: Event lastTimestamp: "2026-05-26T16:15:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:15:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:20Z" name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n.18b329b1a08045a9 namespace: kserve-ci-e2e-test resourceVersion: "16907" uid: 3d3f04c4-a300-485b-9b69-b9d151d78fff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 55cb3473-0b1e-498a-bbcd-70b6660e763b kind: Event lastTimestamp: "2026-05-26T16:15:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:15:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:20Z" name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n.18b329b1a0ec5f5e namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: f4d0e48a-8aa3-425c-af7b-ab5e21c47175 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 16 eventTime: null firstTimestamp: "2026-05-26T16:15:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 55cb3473-0b1e-498a-bbcd-70b6660e763b kind: Event lastTimestamp: "2026-05-26T16:17:44Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:44Z" name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n.18b329b21227d26c namespace: kserve-ci-e2e-test resourceVersion: "18038" uid: 6c9ecdf1-828b-4edb-958c-3c6f1d7ea921 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 55cb3473-0b1e-498a-bbcd-70b6660e763b kind: Event lastTimestamp: "2026-05-26T16:17:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:17:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:36Z" name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n.18b329d13f83b2a6 namespace: kserve-ci-e2e-test resourceVersion: "17929" uid: e7db379b-bee2-47d8-93fb-b62216a5eb9b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 55cb3473-0b1e-498a-bbcd-70b6660e763b kind: Event lastTimestamp: "2026-05-26T16:17:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:17:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:36Z" name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n.18b329d13f843607 namespace: kserve-ci-e2e-test resourceVersion: "17932" uid: 86f976ad-3c81-4421-aae5-13b4ed48145b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 55cb3473-0b1e-498a-bbcd-70b6660e763b kind: Event lastTimestamp: "2026-05-26T16:17:43Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:43Z" name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n.18b329d1bc8a6270 namespace: kserve-ci-e2e-test resourceVersion: "18031" uid: e58c9b00-dbd3-4440-a33d-ff9f1529db3a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5 namespace: kserve-ci-e2e-test resourceVersion: "16806" uid: fa53b70a-8022-4bf3-b523-9fa350ae48d7 kind: Event lastTimestamp: "2026-05-26T16:15:16Z" message: 'Created pod: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5-fwm8n' metadata: creationTimestamp: "2026-05-26T16:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:15:16Z" name: isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5.18b329b098a75dd4 namespace: kserve-ci-e2e-test resourceVersion: "16826" uid: 83043dcc-1cc5-4fed-9e39-628aaeb3f05c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-c3450-predictor namespace: kserve-ci-e2e-test resourceVersion: "16798" uid: f79b4aec-6e7d-4d7d-a5d7-ad7e04da828d kind: Event lastTimestamp: "2026-05-26T16:15:16Z" message: Scaled up replica set isvc-sklearn-scale-raw-c3450-predictor-5c6fc6c8c5 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:15:16Z" name: isvc-sklearn-scale-raw-c3450-predictor.18b329b094b5e685 namespace: kserve-ci-e2e-test resourceVersion: "16810" uid: 45426296-8122-4cb3-a3bb-70113596e1c4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:15:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-c3450-predictor namespace: kserve-ci-e2e-test resourceVersion: "16804" uid: 8d682c51-d20d-4d02-ad60-0ba179b2efcf kind: Event lastTimestamp: "2026-05-26T16:15:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:15:46Z" name: isvc-sklearn-scale-raw-c3450-predictor.18b329b41321a154 namespace: kserve-ci-e2e-test resourceVersion: "17097" uid: 22c7a5c9-9555-4bb1-a462-944fc6165b89 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:15:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-c3450-predictor namespace: kserve-ci-e2e-test resourceVersion: "16804" uid: 8d682c51-d20d-4d02-ad60-0ba179b2efcf kind: Event lastTimestamp: "2026-05-26T16:15:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:15:46Z" name: isvc-sklearn-scale-raw-c3450-predictor.18b329b413969719 namespace: kserve-ci-e2e-test resourceVersion: "17098" uid: 19bcb3b6-ce6b-43d7-bf89-f990f721d7ea reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-26T16:16:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-c3450-predictor namespace: kserve-ci-e2e-test resourceVersion: "16999" uid: 8d682c51-d20d-4d02-ad60-0ba179b2efcf kind: Event lastTimestamp: "2026-05-26T16:17:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:17:31Z" name: isvc-sklearn-scale-raw-c3450-predictor.18b329bb127b1140 namespace: kserve-ci-e2e-test resourceVersion: "17850" uid: 96e815e3-899a-4c40-8e7a-70894917c008 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-26T16:16:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-c3450-predictor namespace: kserve-ci-e2e-test resourceVersion: "16999" uid: 8d682c51-d20d-4d02-ad60-0ba179b2efcf kind: Event lastTimestamp: "2026-05-26T16:17:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:17:31Z" name: isvc-sklearn-scale-raw-c3450-predictor.18b329bb12ffb484 namespace: kserve-ci-e2e-test resourceVersion: "17851" uid: 2c1f30f4-224a-49b2-a23f-42a0cb658d81 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-05-26T16:15:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-c3450 namespace: kserve-ci-e2e-test resourceVersion: "16781" uid: 23ee195b-8732-4aac-93d5-53e2fd4144a0 kind: Event lastTimestamp: "2026-05-26T16:17:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:17:34Z" name: isvc-sklearn-scale-raw-c3450.18b329b09465a49f namespace: kserve-ci-e2e-test resourceVersion: "17884" uid: 69b8031e-5ce2-40bc-b75e-6e4aa1995d30 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-c3450 namespace: kserve-ci-e2e-test resourceVersion: "16781" uid: 23ee195b-8732-4aac-93d5-53e2fd4144a0 kind: Event lastTimestamp: "2026-05-26T16:15:16Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-c3450": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-c3450": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:15:16Z" name: isvc-sklearn-scale-raw-c3450.18b329b094fcdb18 namespace: kserve-ci-e2e-test resourceVersion: "16812" uid: 1321c1b4-5a28-4a74-9e60-bdb6102c6f58 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-c3450 namespace: kserve-ci-e2e-test resourceVersion: "17882" uid: 23ee195b-8732-4aac-93d5-53e2fd4144a0 kind: Event lastTimestamp: "2026-05-26T16:17:34Z" message: InferenceService [isvc-sklearn-scale-raw-c3450] is Ready metadata: creationTimestamp: "2026-05-26T16:17:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:17:34Z" name: isvc-sklearn-scale-raw-c3450.18b329d0e4943bb9 namespace: kserve-ci-e2e-test resourceVersion: "17883" uid: e7942acd-17c8-45df-b508-c72100127f0d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:10:10.379488Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s namespace: kserve-ci-e2e-test resourceVersion: "13975" uid: a35a2f26-8c15-4a9e-b47b-4496e1d8ed8e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s to ip-10-0-137-117.ec2.internal metadata: creationTimestamp: "2026-05-26T16:10:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:10:10Z" name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s.18b3296964b70e47 namespace: kserve-ci-e2e-test resourceVersion: "13981" uid: 541151fb-4b54-42d3-8a9a-360e92a8a754 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8b5dfd5d8-s8tjj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s namespace: kserve-ci-e2e-test resourceVersion: "13987" uid: a35a2f26-8c15-4a9e-b47b-4496e1d8ed8e kind: Event lastTimestamp: "2026-05-26T16:10:10Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:10:10Z" name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s.18b329697e87e0f8 namespace: kserve-ci-e2e-test resourceVersion: "13995" uid: f471209f-6633-421a-a30c-687b1261dea2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: a35a2f26-8c15-4a9e-b47b-4496e1d8ed8e kind: Event lastTimestamp: "2026-05-26T16:10:10Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0413900e48a4ba69f75c45b06af88f792921f4669631dcf3a047c1fe5c1fc341" metadata: creationTimestamp: "2026-05-26T16:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:10Z" name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s.18b32969805cdf0a namespace: kserve-ci-e2e-test resourceVersion: "13997" uid: 105104f0-a990-4be8-af7e-abed1b2205ba reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: a35a2f26-8c15-4a9e-b47b-4496e1d8ed8e kind: Event lastTimestamp: "2026-05-26T16:10:14Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0413900e48a4ba69f75c45b06af88f792921f4669631dcf3a047c1fe5c1fc341" in 3.851s (3.851s including waiting). Image size: 301466548 bytes.' metadata: creationTimestamp: "2026-05-26T16:10:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:14Z" name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s.18b3296a65f5737e namespace: kserve-ci-e2e-test resourceVersion: "14044" uid: 54570734-951c-4977-aea0-f6fa2018c77f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: a35a2f26-8c15-4a9e-b47b-4496e1d8ed8e kind: Event lastTimestamp: "2026-05-26T16:10:14Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T16:10:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:14Z" name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s.18b3296a692ab7c2 namespace: kserve-ci-e2e-test resourceVersion: "14045" uid: b82ebb08-167d-4e2b-871a-130b62350ba8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: a35a2f26-8c15-4a9e-b47b-4496e1d8ed8e kind: Event lastTimestamp: "2026-05-26T16:10:14Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T16:10:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:14Z" name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s.18b3296a699b9850 namespace: kserve-ci-e2e-test resourceVersion: "14046" uid: 62b9fa25-68a1-47ab-9991-96a9b11d1709 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: a35a2f26-8c15-4a9e-b47b-4496e1d8ed8e kind: Event lastTimestamp: "2026-05-26T16:10:18Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-05-26T16:10:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:18Z" name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s.18b3296b64184faf namespace: kserve-ci-e2e-test resourceVersion: "14098" uid: f013b6af-c66a-4ee1-bbf5-f7999427163b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: a35a2f26-8c15-4a9e-b47b-4496e1d8ed8e kind: Event lastTimestamp: "2026-05-26T16:10:38Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.843s (19.843s including waiting). Image size: 1306238203 bytes.' metadata: creationTimestamp: "2026-05-26T16:10:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:38Z" name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s.18b3297002da2f27 namespace: kserve-ci-e2e-test resourceVersion: "14250" uid: a19f158c-00a2-44b0-a137-1f19c3b270d9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: a35a2f26-8c15-4a9e-b47b-4496e1d8ed8e kind: Event lastTimestamp: "2026-05-26T16:10:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:10:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:38Z" name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s.18b3297006218ee1 namespace: kserve-ci-e2e-test resourceVersion: "14251" uid: 325eff2b-e8f9-4958-b933-83a2030b04dd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: a35a2f26-8c15-4a9e-b47b-4496e1d8ed8e kind: Event lastTimestamp: "2026-05-26T16:10:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:10:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:38Z" name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s.18b329700694387d namespace: kserve-ci-e2e-test resourceVersion: "14252" uid: 4c6d7288-374f-48db-97bd-2ec2d56d8ddd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: a35a2f26-8c15-4a9e-b47b-4496e1d8ed8e kind: Event lastTimestamp: "2026-05-26T16:10:38Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-26T16:10:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:38Z" name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s.18b3297006a15387 namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: 05d9b436-d110-4b61-a967-0f3ca7ced3d2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: a35a2f26-8c15-4a9e-b47b-4496e1d8ed8e kind: Event lastTimestamp: "2026-05-26T16:10:41Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.706s (2.707s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-26T16:10:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:41Z" name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s.18b32970a7fb4436 namespace: kserve-ci-e2e-test resourceVersion: "14284" uid: 532c18cf-d526-494e-8360-fcfc3d0e2c79 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: a35a2f26-8c15-4a9e-b47b-4496e1d8ed8e kind: Event lastTimestamp: "2026-05-26T16:10:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:10:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:41Z" name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s.18b32970ab2622e4 namespace: kserve-ci-e2e-test resourceVersion: "14285" uid: ce51c5e5-3486-446b-8164-7b988b820682 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: a35a2f26-8c15-4a9e-b47b-4496e1d8ed8e kind: Event lastTimestamp: "2026-05-26T16:10:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:10:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:10:41Z" name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s.18b32970aba2fea7 namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: bacaacb7-5492-4185-acbb-383430c03508 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-26T16:10:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: a35a2f26-8c15-4a9e-b47b-4496e1d8ed8e kind: Event lastTimestamp: "2026-05-26T16:11:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:10:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:29Z" name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s.18b32970ffb3bb78 namespace: kserve-ci-e2e-test resourceVersion: "14639" uid: 71bb0415-5522-47d2-9e8e-b092aeff92c2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: a35a2f26-8c15-4a9e-b47b-4496e1d8ed8e kind: Event lastTimestamp: "2026-05-26T16:11:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:50Z" name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s.18b32980af65aef4 namespace: kserve-ci-e2e-test resourceVersion: "14889" uid: 60c9b355-6227-49cb-848d-20fd683afe3e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: a35a2f26-8c15-4a9e-b47b-4496e1d8ed8e kind: Event lastTimestamp: "2026-05-26T16:11:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:50Z" name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s.18b32980af663c1c namespace: kserve-ci-e2e-test resourceVersion: "14892" uid: 09652e7a-cdbf-4cdb-840f-d900bd09bc34 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: a35a2f26-8c15-4a9e-b47b-4496e1d8ed8e kind: Event lastTimestamp: "2026-05-26T16:11:54Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:54Z" name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s.18b3298187bf3a62 namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: 58b20ca6-dea9-4ddf-9a9f-7844828070f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969 namespace: kserve-ci-e2e-test resourceVersion: "13960" uid: 38d5a74b-a546-45ea-bd55-fdb76a2e2be5 kind: Event lastTimestamp: "2026-05-26T16:10:10Z" message: 'Created pod: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969-2785s' metadata: creationTimestamp: "2026-05-26T16:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:10:10Z" name: isvc-xgboost-graph-raw-5c32b-predictor-76bb944969.18b3296963ff9c33 namespace: kserve-ci-e2e-test resourceVersion: "13977" uid: 6a2ab3d5-dbf8-4f2c-9216-0092d63afa78 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-5c32b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13956" uid: d02925c9-8e46-42dc-bb64-d14545bb0ac2 kind: Event lastTimestamp: "2026-05-26T16:10:10Z" message: Scaled up replica set isvc-xgboost-graph-raw-5c32b-predictor-76bb944969 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:10:10Z" name: isvc-xgboost-graph-raw-5c32b-predictor.18b329695bf0d096 namespace: kserve-ci-e2e-test resourceVersion: "13966" uid: 5d263fef-cd24-4cd9-8172-45ef2136448b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:10:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-5c32b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13957" uid: 87aff3bc-2dab-489e-8fc1-adc4398517e7 kind: Event lastTimestamp: "2026-05-26T16:10:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:10:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:10:40Z" name: isvc-xgboost-graph-raw-5c32b-predictor.18b3296cd9f0ab6d namespace: kserve-ci-e2e-test resourceVersion: "14269" uid: 32c1f2cd-60aa-4d57-afc4-e787c28c6dc9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:10:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-5c32b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13957" uid: 87aff3bc-2dab-489e-8fc1-adc4398517e7 kind: Event lastTimestamp: "2026-05-26T16:10:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:10:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:10:40Z" name: isvc-xgboost-graph-raw-5c32b-predictor.18b3296cda74ce23 namespace: kserve-ci-e2e-test resourceVersion: "14270" uid: 67d6a9d2-2d6d-4342-9898-15ffcb831d43 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T16:10:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-5c32b-predictor namespace: kserve-ci-e2e-test resourceVersion: "14152" uid: 87aff3bc-2dab-489e-8fc1-adc4398517e7 kind: Event lastTimestamp: "2026-05-26T16:11:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:10:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:11:40Z" name: isvc-xgboost-graph-raw-5c32b-predictor.18b32973d84b0e73 namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: 8363f722-667b-4b56-9e78-3327477f60fb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T16:10:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-5c32b-predictor namespace: kserve-ci-e2e-test resourceVersion: "14152" uid: 87aff3bc-2dab-489e-8fc1-adc4398517e7 kind: Event lastTimestamp: "2026-05-26T16:11:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:10:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:11:40Z" name: isvc-xgboost-graph-raw-5c32b-predictor.18b32973d901cd08 namespace: kserve-ci-e2e-test resourceVersion: "14764" uid: 8109d6c6-0a18-4133-8997-b1e962717d62 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-05-26T16:10:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-5c32b namespace: kserve-ci-e2e-test resourceVersion: "13938" uid: 5ff60030-1919-4ea8-8700-4da40212b6a7 kind: Event lastTimestamp: "2026-05-26T16:11:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:11:39Z" name: isvc-xgboost-graph-raw-5c32b.18b329695b18599f namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: d21836de-20f1-476e-9c5e-becf2d480b3c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:10:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-5c32b namespace: kserve-ci-e2e-test resourceVersion: "13938" uid: 5ff60030-1919-4ea8-8700-4da40212b6a7 kind: Event lastTimestamp: "2026-05-26T16:10:10Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-5c32b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-5c32b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:10:10Z" name: isvc-xgboost-graph-raw-5c32b.18b329695beddc95 namespace: kserve-ci-e2e-test resourceVersion: "13964" uid: a613f0c0-9c18-4fda-aea9-068a7c856d4f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-5c32b namespace: kserve-ci-e2e-test resourceVersion: "14721" uid: 5ff60030-1919-4ea8-8700-4da40212b6a7 kind: Event lastTimestamp: "2026-05-26T16:11:39Z" message: InferenceService [isvc-xgboost-graph-raw-5c32b] is Ready metadata: creationTimestamp: "2026-05-26T16:11:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:11:39Z" name: isvc-xgboost-graph-raw-5c32b.18b3297e1113132d namespace: kserve-ci-e2e-test resourceVersion: "14722" uid: 230beef5-37b6-4fb3-b8fa-30a49af9d643 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:11:50.701271Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24 namespace: kserve-ci-e2e-test resourceVersion: "14952" uid: 3723d817-ba42-4581-8d77-1738ecf4b6ca kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24 to ip-10-0-137-117.ec2.internal metadata: creationTimestamp: "2026-05-26T16:11:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:11:50Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24.18b32980c05be0a2 namespace: kserve-ci-e2e-test resourceVersion: "14957" uid: 2c634dbd-7e47-4351-955e-5a6bd4ba2c23 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8b5dfd5d8-s8tjj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 3723d817-ba42-4581-8d77-1738ecf4b6ca kind: Event lastTimestamp: "2026-05-26T16:11:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-hpa-89f63-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:50Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24.18b32980cb7315a1 namespace: kserve-ci-e2e-test resourceVersion: "14967" uid: 1515f589-424c-4836-9d21-78025510e424 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24 namespace: kserve-ci-e2e-test resourceVersion: "14964" uid: 3723d817-ba42-4581-8d77-1738ecf4b6ca kind: Event lastTimestamp: "2026-05-26T16:11:51Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:11:51Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24.18b32980fd769729 namespace: kserve-ci-e2e-test resourceVersion: "14983" uid: 4961b6f0-c517-45b8-8deb-f76d1f2c6b0b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 3723d817-ba42-4581-8d77-1738ecf4b6ca kind: Event lastTimestamp: "2026-05-26T16:11:51Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0413900e48a4ba69f75c45b06af88f792921f4669631dcf3a047c1fe5c1fc341" already present on machine metadata: creationTimestamp: "2026-05-26T16:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:51Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24.18b32980ff2f8329 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: ed2ab19d-a350-4a3d-a41f-7843519314ff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 3723d817-ba42-4581-8d77-1738ecf4b6ca kind: Event lastTimestamp: "2026-05-26T16:11:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T16:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:51Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24.18b32981024af88f namespace: kserve-ci-e2e-test resourceVersion: "14988" uid: f8d76508-9563-40ea-bb3b-9d158b592c06 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 3723d817-ba42-4581-8d77-1738ecf4b6ca kind: Event lastTimestamp: "2026-05-26T16:11:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T16:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:51Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24.18b3298102bf8eca namespace: kserve-ci-e2e-test resourceVersion: "14989" uid: 1d866e38-daaf-49b8-ab94-4925c149457b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 3723d817-ba42-4581-8d77-1738ecf4b6ca kind: Event lastTimestamp: "2026-05-26T16:11:56Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-05-26T16:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:56Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24.18b329820c6408cc namespace: kserve-ci-e2e-test resourceVersion: "15057" uid: cd014db2-b61d-47d7-98ed-cdbb9d52aee4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 3723d817-ba42-4581-8d77-1738ecf4b6ca kind: Event lastTimestamp: "2026-05-26T16:11:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:56Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24.18b329820f7c4e9d namespace: kserve-ci-e2e-test resourceVersion: "15060" uid: 3ac8e53d-d701-49ee-859d-9e8c4df18de0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 3723d817-ba42-4581-8d77-1738ecf4b6ca kind: Event lastTimestamp: "2026-05-26T16:11:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:56Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24.18b329820fefa820 namespace: kserve-ci-e2e-test resourceVersion: "15061" uid: 6b280cc0-2323-429a-b6f2-615825dd077c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 3723d817-ba42-4581-8d77-1738ecf4b6ca kind: Event lastTimestamp: "2026-05-26T16:11:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:56Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24.18b329820ffc9a60 namespace: kserve-ci-e2e-test resourceVersion: "15062" uid: c264fe9e-d171-4d9e-8f86-954319d122f7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 3723d817-ba42-4581-8d77-1738ecf4b6ca kind: Event lastTimestamp: "2026-05-26T16:11:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:56Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24.18b32982133a1022 namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: c8257187-944b-41c6-85b9-87b1605536dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 3723d817-ba42-4581-8d77-1738ecf4b6ca kind: Event lastTimestamp: "2026-05-26T16:11:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:56Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24.18b3298213adc628 namespace: kserve-ci-e2e-test resourceVersion: "15066" uid: 664a4e3c-e374-4b96-adcc-57621eacb210 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T16:11:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 3723d817-ba42-4581-8d77-1738ecf4b6ca kind: Event lastTimestamp: "2026-05-26T16:13:34Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:11:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:34Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24.18b329828435c9b9 namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: 5c142242-9f82-4e46-a6fc-02e5b2cc6b70 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 3723d817-ba42-4581-8d77-1738ecf4b6ca kind: Event lastTimestamp: "2026-05-26T16:13:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:30Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24.18b3299812cd7343 namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: 6233e077-1b33-4534-af0e-4c60ab72f245 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 3723d817-ba42-4581-8d77-1738ecf4b6ca kind: Event lastTimestamp: "2026-05-26T16:13:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:30Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24.18b3299812cde93e namespace: kserve-ci-e2e-test resourceVersion: "15902" uid: e882dd74-008c-4d37-addb-f62b11c4bdc4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 3723d817-ba42-4581-8d77-1738ecf4b6ca kind: Event lastTimestamp: "2026-05-26T16:13:34Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:34Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24.18b32998de68f8d0 namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: 58444cb2-8e87-49ee-8bf4-fe95b54aeecb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf namespace: kserve-ci-e2e-test resourceVersion: "14943" uid: a5696ce1-72e8-4298-9bbe-e034a0c9ccb2 kind: Event lastTimestamp: "2026-05-26T16:11:50Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf-rxj24' metadata: creationTimestamp: "2026-05-26T16:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:11:50Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf.18b32980bfdffb04 namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 2a5cf478-c417-4f4d-adaa-33c0eddd44ee reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-89f63-predictor namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: e0d40eee-9c72-498d-8baf-77bdda38a0c2 kind: Event lastTimestamp: "2026-05-26T16:11:50Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-89f63-predictor-747dddcbf from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:11:50Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor.18b32980bc4ca69c namespace: kserve-ci-e2e-test resourceVersion: "14947" uid: 92df4fb4-2fb8-4722-a6c2-78b3416c3a04 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:12:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-89f63-predictor namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: 45c12135-d8bc-4a2d-85a5-87cfca22ab43 kind: Event lastTimestamp: "2026-05-26T16:12:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:12:35Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor.18b329843ae0e656 namespace: kserve-ci-e2e-test resourceVersion: "15373" uid: 569e0492-f51d-43a1-9b06-742783c57ac9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:12:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-89f63-predictor namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: 45c12135-d8bc-4a2d-85a5-87cfca22ab43 kind: Event lastTimestamp: "2026-05-26T16:12:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:12:35Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor.18b329843b5c63dd namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: aaf7845e-f7a9-4998-aa01-8f0eb9706047 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:12:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-89f63-predictor namespace: kserve-ci-e2e-test resourceVersion: "15150" uid: 45c12135-d8bc-4a2d-85a5-87cfca22ab43 kind: Event lastTimestamp: "2026-05-26T16:13:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:12:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:13:20Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor.18b3298eb7f8d3a2 namespace: kserve-ci-e2e-test resourceVersion: "15745" uid: 38df3aa8-bef5-4a61-b3c5-9ff27481eb31 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:12:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-89f63-predictor namespace: kserve-ci-e2e-test resourceVersion: "15150" uid: 45c12135-d8bc-4a2d-85a5-87cfca22ab43 kind: Event lastTimestamp: "2026-05-26T16:13:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:12:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:13:20Z" name: isvc-xgboost-graph-raw-hpa-89f63-predictor.18b3298eb87fd111 namespace: kserve-ci-e2e-test resourceVersion: "15746" uid: b599c8e1-db4b-40a1-a1af-284397692b96 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T16:11:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-89f63 namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: 56b7ff8e-2595-4e45-ba99-059232e93918 kind: Event lastTimestamp: "2026-05-26T16:13:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:13:04Z" name: isvc-xgboost-graph-raw-hpa-89f63.18b32980bc1532d8 namespace: kserve-ci-e2e-test resourceVersion: "15594" uid: b70bbf0b-38a4-4f60-b41c-299dd9a89c08 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-89f63 namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: 56b7ff8e-2595-4e45-ba99-059232e93918 kind: Event lastTimestamp: "2026-05-26T16:11:50Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-89f63": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-89f63": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:11:50Z" name: isvc-xgboost-graph-raw-hpa-89f63.18b32980bceed939 namespace: kserve-ci-e2e-test resourceVersion: "14949" uid: 79224b92-f3ff-44b3-b173-a3e8183d6820 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-89f63 namespace: kserve-ci-e2e-test resourceVersion: "15592" uid: 56b7ff8e-2595-4e45-ba99-059232e93918 kind: Event lastTimestamp: "2026-05-26T16:13:04Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-89f63] is Ready metadata: creationTimestamp: "2026-05-26T16:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:13:04Z" name: isvc-xgboost-graph-raw-hpa-89f63.18b32991e8d83f46 namespace: kserve-ci-e2e-test resourceVersion: "15593" uid: b1e1af87-30d6-41a9-9e63-5f478acaca00 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:13:30.858854Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: 95eb1efb-8eec-4333-b8b3-0dd730ea8532 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz to ip-10-0-134-19.ec2.internal metadata: creationTimestamp: "2026-05-26T16:13:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:13:30Z" name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz.18b3299812374b5f namespace: kserve-ci-e2e-test resourceVersion: "15897" uid: 8610a34f-a226-46c9-946c-59601c255b5c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8b5dfd5d8-s8tjj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:31Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz namespace: kserve-ci-e2e-test resourceVersion: "15904" uid: 95eb1efb-8eec-4333-b8b3-0dd730ea8532 kind: Event lastTimestamp: "2026-05-26T16:13:31Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:13:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:13:31Z" name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz.18b329982b6707fd namespace: kserve-ci-e2e-test resourceVersion: "15914" uid: 489e6de2-6c77-4743-a7f3-f7230cfabe63 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 95eb1efb-8eec-4333-b8b3-0dd730ea8532 kind: Event lastTimestamp: "2026-05-26T16:13:31Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-05-26T16:13:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:31Z" name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz.18b329982ccc3bc3 namespace: kserve-ci-e2e-test resourceVersion: "15916" uid: 92cc294c-c781-4636-9239-e3122384b89c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 95eb1efb-8eec-4333-b8b3-0dd730ea8532 kind: Event lastTimestamp: "2026-05-26T16:13:32Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 992ms (992ms including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-05-26T16:13:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:32Z" name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz.18b3299867f17ca8 namespace: kserve-ci-e2e-test resourceVersion: "15929" uid: ed54eef0-d801-4ebc-a373-8f7ad29a488d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 95eb1efb-8eec-4333-b8b3-0dd730ea8532 kind: Event lastTimestamp: "2026-05-26T16:13:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:13:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:32Z" name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz.18b329986a8bd728 namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: cff78772-3c88-4073-b888-dc8109661f45 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 95eb1efb-8eec-4333-b8b3-0dd730ea8532 kind: Event lastTimestamp: "2026-05-26T16:13:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:13:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:32Z" name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz.18b329986afbb8bc namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: 5226ea23-2abb-4146-83ad-247f0ddc598d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 95eb1efb-8eec-4333-b8b3-0dd730ea8532 kind: Event lastTimestamp: "2026-05-26T16:13:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:13:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:32Z" name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz.18b329986b076569 namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: 40165b7a-8032-4956-91be-1a4a48ae2e63 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 95eb1efb-8eec-4333-b8b3-0dd730ea8532 kind: Event lastTimestamp: "2026-05-26T16:13:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:13:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:32Z" name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz.18b329986db2fe8e namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: d269269f-f4b0-4959-ac1d-5fef7ff99e92 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 95eb1efb-8eec-4333-b8b3-0dd730ea8532 kind: Event lastTimestamp: "2026-05-26T16:13:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:13:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:32Z" name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz.18b329986e1e5113 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 147dde14-d78d-4cd9-b1a8-398327d9aff0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 95eb1efb-8eec-4333-b8b3-0dd730ea8532 kind: Event lastTimestamp: "2026-05-26T16:15:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:16Z" name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz.18b329b0a0681bf6 namespace: kserve-ci-e2e-test resourceVersion: "16851" uid: eac6957a-1664-47f1-bde0-202743553842 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 95eb1efb-8eec-4333-b8b3-0dd730ea8532 kind: Event lastTimestamp: "2026-05-26T16:15:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:16Z" name: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz.18b329b0a0693ba8 namespace: kserve-ci-e2e-test resourceVersion: "16853" uid: 70915ce2-b36a-453c-8ed4-cf76db6d51e3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-4b426-predictor-75c485b4b4 namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: 6d9faf44-2cd3-4eba-8642-507afce7eb09 kind: Event lastTimestamp: "2026-05-26T16:13:30Z" message: 'Created pod: message-dumper-raw-4b426-predictor-75c485b4b4-zwcxz' metadata: creationTimestamp: "2026-05-26T16:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:13:30Z" name: message-dumper-raw-4b426-predictor-75c485b4b4.18b3299811b14227 namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: 55caacc1-3a95-432e-860a-3c2403841b94 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-4b426-predictor namespace: kserve-ci-e2e-test resourceVersion: "15864" uid: b7dc3654-3d35-40e2-872e-960a6f658437 kind: Event lastTimestamp: "2026-05-26T16:13:30Z" message: Scaled up replica set message-dumper-raw-4b426-predictor-75c485b4b4 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:13:30Z" name: message-dumper-raw-4b426-predictor.18b329980e228672 namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: b94ccd1a-28c6-41bd-92a5-c190b8211fcb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-4b426-predictor namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: 58d0e993-37f7-4c43-b728-ff6b65b0570a kind: Event lastTimestamp: "2026-05-26T16:13:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:13:45Z" name: message-dumper-raw-4b426-predictor.18b3299b8d4b6a0a namespace: kserve-ci-e2e-test resourceVersion: "16057" uid: d6d248b7-3c66-4cfa-8877-ea61bfc905d5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-4b426-predictor namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: 58d0e993-37f7-4c43-b728-ff6b65b0570a kind: Event lastTimestamp: "2026-05-26T16:13:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:13:45Z" name: message-dumper-raw-4b426-predictor.18b3299b8db711f4 namespace: kserve-ci-e2e-test resourceVersion: "16059" uid: 8a793451-a0ff-48ca-bc23-bf7a488959b3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-4b426-predictor namespace: kserve-ci-e2e-test resourceVersion: "16058" uid: 58d0e993-37f7-4c43-b728-ff6b65b0570a kind: Event lastTimestamp: "2026-05-26T16:14:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:14:00Z" name: message-dumper-raw-4b426-predictor.18b3299f0c9cf70f namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: d688b856-c07b-4be2-8405-91bbc22708dc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-4b426-predictor namespace: kserve-ci-e2e-test resourceVersion: "16058" uid: 58d0e993-37f7-4c43-b728-ff6b65b0570a kind: Event lastTimestamp: "2026-05-26T16:14:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:14:00Z" name: message-dumper-raw-4b426-predictor.18b3299f0d141273 namespace: kserve-ci-e2e-test resourceVersion: "16232" uid: 5e1b967f-e236-4f9d-bea6-c27b865d10d7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-26T16:13:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-4b426 namespace: kserve-ci-e2e-test resourceVersion: "15842" uid: f2076557-a228-4066-aac4-bc82249fb924 kind: Event lastTimestamp: "2026-05-26T16:13:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:13:41Z" name: message-dumper-raw-4b426.18b329980e27efb2 namespace: kserve-ci-e2e-test resourceVersion: "16020" uid: 1550f0d0-3e00-435f-acf3-e24268816782 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:13:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-4b426 namespace: kserve-ci-e2e-test resourceVersion: "15842" uid: f2076557-a228-4066-aac4-bc82249fb924 kind: Event lastTimestamp: "2026-05-26T16:13:30Z" message: 'Failed to update status for InferenceService "message-dumper-raw-4b426": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-4b426": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:13:30Z" name: message-dumper-raw-4b426.18b329980eaee617 namespace: kserve-ci-e2e-test resourceVersion: "15907" uid: 29741114-7ec5-455b-9f81-85d7c3064ff7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-4b426 namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: f2076557-a228-4066-aac4-bc82249fb924 kind: Event lastTimestamp: "2026-05-26T16:13:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-4b426": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:13:30Z" name: message-dumper-raw-4b426.18b3299815af7f10 namespace: kserve-ci-e2e-test resourceVersion: "15909" uid: 255e46ab-ce9d-41d0-a963-9eefcd4389a9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-4b426 namespace: kserve-ci-e2e-test resourceVersion: "16018" uid: f2076557-a228-4066-aac4-bc82249fb924 kind: Event lastTimestamp: "2026-05-26T16:13:41Z" message: InferenceService [message-dumper-raw-4b426] is Ready metadata: creationTimestamp: "2026-05-26T16:13:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:13:41Z" name: message-dumper-raw-4b426.18b3299a7e9f2b6a namespace: kserve-ci-e2e-test resourceVersion: "16019" uid: 922cd21c-8165-463e-86ef-674a921af24f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:11:40.199623Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-5c32b-6759b7bc5d-tlhr6 namespace: kserve-ci-e2e-test resourceVersion: "14746" uid: 52c680e5-fe60-43c0-ab43-61eff5761b4a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-5c32b-6759b7bc5d-tlhr6 to ip-10-0-137-117.ec2.internal metadata: creationTimestamp: "2026-05-26T16:11:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:11:40Z" name: model-chainer-raw-5c32b-6759b7bc5d-tlhr6.18b3297e4e697ddf namespace: kserve-ci-e2e-test resourceVersion: "14753" uid: f100da5f-d8f4-4703-b27a-b900010ed4e9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8b5dfd5d8-s8tjj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:40Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-5c32b-6759b7bc5d-tlhr6 namespace: kserve-ci-e2e-test resourceVersion: "14757" uid: 52c680e5-fe60-43c0-ab43-61eff5761b4a kind: Event lastTimestamp: "2026-05-26T16:11:40Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:11:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:11:40Z" name: model-chainer-raw-5c32b-6759b7bc5d-tlhr6.18b3297e67c4f74f namespace: kserve-ci-e2e-test resourceVersion: "14769" uid: 542c5192-1289-41d9-9536-faba585ab185 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-5c32b} kind: Pod name: model-chainer-raw-5c32b-6759b7bc5d-tlhr6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 52c680e5-fe60-43c0-ab43-61eff5761b4a kind: Event lastTimestamp: "2026-05-26T16:11:40Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:82e40282a2f2a4472a45c41fa18200d967bc642df3ede2a9ddc4e0694f6a1107" metadata: creationTimestamp: "2026-05-26T16:11:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:40Z" name: model-chainer-raw-5c32b-6759b7bc5d-tlhr6.18b3297e6947de81 namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: db503c29-7d7f-4e8d-8395-9ec395dddede reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-5c32b} kind: Pod name: model-chainer-raw-5c32b-6759b7bc5d-tlhr6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 52c680e5-fe60-43c0-ab43-61eff5761b4a kind: Event lastTimestamp: "2026-05-26T16:11:43Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:82e40282a2f2a4472a45c41fa18200d967bc642df3ede2a9ddc4e0694f6a1107" in 2.415s (2.415s including waiting). Image size: 217997401 bytes.' metadata: creationTimestamp: "2026-05-26T16:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:43Z" name: model-chainer-raw-5c32b-6759b7bc5d-tlhr6.18b3297ef944657e namespace: kserve-ci-e2e-test resourceVersion: "14789" uid: 9d287b3e-1af4-43f4-a625-472f2efcce15 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-5c32b} kind: Pod name: model-chainer-raw-5c32b-6759b7bc5d-tlhr6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 52c680e5-fe60-43c0-ab43-61eff5761b4a kind: Event lastTimestamp: "2026-05-26T16:11:43Z" message: 'Created container: model-chainer-raw-5c32b' metadata: creationTimestamp: "2026-05-26T16:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:43Z" name: model-chainer-raw-5c32b-6759b7bc5d-tlhr6.18b3297efcbec802 namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: 8ea9b46c-b2aa-4f64-914d-83c92d7c8ca0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-5c32b} kind: Pod name: model-chainer-raw-5c32b-6759b7bc5d-tlhr6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 52c680e5-fe60-43c0-ab43-61eff5761b4a kind: Event lastTimestamp: "2026-05-26T16:11:43Z" message: Started container model-chainer-raw-5c32b metadata: creationTimestamp: "2026-05-26T16:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:43Z" name: model-chainer-raw-5c32b-6759b7bc5d-tlhr6.18b3297efd327514 namespace: kserve-ci-e2e-test resourceVersion: "14791" uid: dc6a6151-37b4-44cf-9781-342e912f7fa0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-5c32b} kind: Pod name: model-chainer-raw-5c32b-6759b7bc5d-tlhr6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 52c680e5-fe60-43c0-ab43-61eff5761b4a kind: Event lastTimestamp: "2026-05-26T16:11:50Z" message: Stopping container model-chainer-raw-5c32b metadata: creationTimestamp: "2026-05-26T16:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:11:50Z" name: model-chainer-raw-5c32b-6759b7bc5d-tlhr6.18b32980a3b3f2a0 namespace: kserve-ci-e2e-test resourceVersion: "14863" uid: 1d1970d2-9101-4b16-9cfd-fab6d2969131 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:11:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-5c32b} kind: Pod name: model-chainer-raw-5c32b-6759b7bc5d-tlhr6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 52c680e5-fe60-43c0-ab43-61eff5761b4a kind: Event lastTimestamp: "2026-05-26T16:12:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T16:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:12:19Z" name: model-chainer-raw-5c32b-6759b7bc5d-tlhr6.18b3298192bbd6eb namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 1404acc5-d3c9-4e13-a4a8-392053ce3aa1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-5c32b-6759b7bc5d namespace: kserve-ci-e2e-test resourceVersion: "14741" uid: 3cc32016-dd56-413c-a932-e478574b674a kind: Event lastTimestamp: "2026-05-26T16:11:40Z" message: 'Created pod: model-chainer-raw-5c32b-6759b7bc5d-tlhr6' metadata: creationTimestamp: "2026-05-26T16:11:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:11:40Z" name: model-chainer-raw-5c32b-6759b7bc5d.18b3297e4d00784a namespace: kserve-ci-e2e-test resourceVersion: "14748" uid: 17c2269a-dd2a-41db-a473-6be77f9f6ce8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-5c32b namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 28f28a98-8db3-4e13-bd96-c2c30eb0d6b1 kind: Event lastTimestamp: "2026-05-26T16:11:40Z" message: Scaled up replica set model-chainer-raw-5c32b-6759b7bc5d from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:11:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:11:40Z" name: model-chainer-raw-5c32b.18b3297e4b83a412 namespace: kserve-ci-e2e-test resourceVersion: "14743" uid: 7b3c6583-7180-4c21-ac8e-5c8640b0162e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-5c32b namespace: kserve-ci-e2e-test resourceVersion: "14760" uid: 000ce9c9-a19f-4ef6-a859-0f5d48285fa4 kind: Event lastTimestamp: "2026-05-26T16:11:40Z" message: InferenceGraph [model-chainer-raw-5c32b] is Ready metadata: creationTimestamp: "2026-05-26T16:11:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:11:40Z" name: model-chainer-raw-5c32b.18b3297e54c968e3 namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: a5950072-6f4a-4727-a9e8-47cfe4931f8f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-5c32b namespace: kserve-ci-e2e-test resourceVersion: "14732" uid: 000ce9c9-a19f-4ef6-a859-0f5d48285fa4 kind: Event lastTimestamp: "2026-05-26T16:11:40Z" message: 'Failed to update status for InferenceGraph "model-chainer-raw-5c32b": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-5c32b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:11:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:11:40Z" name: model-chainer-raw-5c32b.18b3297e59633d14 namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: ae4ca650-183d-40e3-a32d-87916c3c78bd reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:11:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-5c32b namespace: kserve-ci-e2e-test resourceVersion: "14732" uid: 000ce9c9-a19f-4ef6-a859-0f5d48285fa4 kind: Event lastTimestamp: "2026-05-26T16:11:40Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-5c32b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:11:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:11:40Z" name: model-chainer-raw-5c32b.18b3297e59638a1a namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: d9987f79-084d-464f-b135-8be501a764a2 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:13:20.561642Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-89f63-58f5997b9d-q9zwh namespace: kserve-ci-e2e-test resourceVersion: "15725" uid: 7493decd-7537-40b4-8f04-247dd339dfac kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-89f63-58f5997b9d-q9zwh to ip-10-0-137-117.ec2.internal metadata: creationTimestamp: "2026-05-26T16:13:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:13:20Z" name: model-chainer-raw-hpa-89f63-58f5997b9d-q9zwh.18b32995ac744f45 namespace: kserve-ci-e2e-test resourceVersion: "15732" uid: 2727aedb-2801-43c2-947f-5e8e826f6e19 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8b5dfd5d8-s8tjj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:20Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-89f63-58f5997b9d-q9zwh namespace: kserve-ci-e2e-test resourceVersion: "15734" uid: 7493decd-7537-40b4-8f04-247dd339dfac kind: Event lastTimestamp: "2026-05-26T16:13:20Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:13:20Z" name: model-chainer-raw-hpa-89f63-58f5997b9d-q9zwh.18b32995c5b17e96 namespace: kserve-ci-e2e-test resourceVersion: "15747" uid: 42311a6f-0cfe-4edb-b89b-cc6c74f4f877 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-89f63} kind: Pod name: model-chainer-raw-hpa-89f63-58f5997b9d-q9zwh namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 7493decd-7537-40b4-8f04-247dd339dfac kind: Event lastTimestamp: "2026-05-26T16:13:21Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:82e40282a2f2a4472a45c41fa18200d967bc642df3ede2a9ddc4e0694f6a1107" already present on machine metadata: creationTimestamp: "2026-05-26T16:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:21Z" name: model-chainer-raw-hpa-89f63-58f5997b9d-q9zwh.18b32995c7228c15 namespace: kserve-ci-e2e-test resourceVersion: "15750" uid: 6045144d-028f-41cc-bf42-63f3fba6316f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-89f63} kind: Pod name: model-chainer-raw-hpa-89f63-58f5997b9d-q9zwh namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 7493decd-7537-40b4-8f04-247dd339dfac kind: Event lastTimestamp: "2026-05-26T16:13:21Z" message: 'Created container: model-chainer-raw-hpa-89f63' metadata: creationTimestamp: "2026-05-26T16:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:21Z" name: model-chainer-raw-hpa-89f63-58f5997b9d-q9zwh.18b32995ca286af1 namespace: kserve-ci-e2e-test resourceVersion: "15751" uid: 21de9287-6a32-4563-8b7b-f88bd2e6c552 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-89f63} kind: Pod name: model-chainer-raw-hpa-89f63-58f5997b9d-q9zwh namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 7493decd-7537-40b4-8f04-247dd339dfac kind: Event lastTimestamp: "2026-05-26T16:13:21Z" message: Started container model-chainer-raw-hpa-89f63 metadata: creationTimestamp: "2026-05-26T16:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:21Z" name: model-chainer-raw-hpa-89f63-58f5997b9d-q9zwh.18b32995cad98a9e namespace: kserve-ci-e2e-test resourceVersion: "15752" uid: 5f309f7e-699d-41b7-9818-eb408a7dd408 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-89f63} kind: Pod name: model-chainer-raw-hpa-89f63-58f5997b9d-q9zwh namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 7493decd-7537-40b4-8f04-247dd339dfac kind: Event lastTimestamp: "2026-05-26T16:13:30Z" message: Stopping container model-chainer-raw-hpa-89f63 metadata: creationTimestamp: "2026-05-26T16:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:30Z" name: model-chainer-raw-hpa-89f63-58f5997b9d-q9zwh.18b3299803ef12dd namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 8002f506-06b0-4f38-a15c-ad7ff40a9aee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:13:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-89f63} kind: Pod name: model-chainer-raw-hpa-89f63-58f5997b9d-q9zwh namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 7493decd-7537-40b4-8f04-247dd339dfac kind: Event lastTimestamp: "2026-05-26T16:13:57Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T16:13:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:13:57Z" name: model-chainer-raw-hpa-89f63-58f5997b9d-q9zwh.18b3299876b3dbf6 namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: 92b0a206-407e-4219-a459-68509fe07ea4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-117.ec2.internal source: component: kubelet host: ip-10-0-137-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-89f63-58f5997b9d namespace: kserve-ci-e2e-test resourceVersion: "15722" uid: 697cabc4-a01c-4785-ba7f-a0941c2337e9 kind: Event lastTimestamp: "2026-05-26T16:13:20Z" message: 'Created pod: model-chainer-raw-hpa-89f63-58f5997b9d-q9zwh' metadata: creationTimestamp: "2026-05-26T16:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:13:20Z" name: model-chainer-raw-hpa-89f63-58f5997b9d.18b32995ab848d96 namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: 16a8eb8d-e235-4e6f-959b-3fdcaaf9abec reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-89f63 namespace: kserve-ci-e2e-test resourceVersion: "15720" uid: 6b9ffcb1-7e0d-44a1-ad3e-b9a26829d902 kind: Event lastTimestamp: "2026-05-26T16:13:20Z" message: Scaled up replica set model-chainer-raw-hpa-89f63-58f5997b9d from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:13:20Z" name: model-chainer-raw-hpa-89f63.18b32995a9f317fc namespace: kserve-ci-e2e-test resourceVersion: "15723" uid: ef7942de-5f91-47d4-b34a-bd82ea687b22 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:13:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-89f63 namespace: kserve-ci-e2e-test resourceVersion: "15739" uid: 50416e53-9053-4440-b0d6-74c3cd5c8b81 kind: Event lastTimestamp: "2026-05-26T16:13:20Z" message: InferenceGraph [model-chainer-raw-hpa-89f63] is Ready metadata: creationTimestamp: "2026-05-26T16:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:13:20Z" name: model-chainer-raw-hpa-89f63.18b32995b0b6442a namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: 9299f01e-2145-4880-a43a-1e46fc8127ae reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:19:25.160420Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-ea140-predictor-85487667c-ztpzz namespace: kserve-ci-e2e-test resourceVersion: "18985" uid: 3b1ddd12-bc61-4aff-9907-b08cd577ee3a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-ea140-predictor-85487667c-ztpzz to ip-10-0-134-19.ec2.internal metadata: creationTimestamp: "2026-05-26T16:19:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:19:25Z" name: raw-sklearn-ea140-predictor-85487667c-ztpzz.18b329ea903c2c25 namespace: kserve-ci-e2e-test resourceVersion: "18989" uid: c3bbd866-9892-4c3b-b8a0-31c7996392e5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8b5dfd5d8-s8tjj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:25Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-ea140-predictor-85487667c-ztpzz namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: 3b1ddd12-bc61-4aff-9907-b08cd577ee3a kind: Event lastTimestamp: "2026-05-26T16:19:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-ea140-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:19:25Z" name: raw-sklearn-ea140-predictor-85487667c-ztpzz.18b329ea9ab77467 namespace: kserve-ci-e2e-test resourceVersion: "18997" uid: f6146507-9a25-4f61-a1ff-f4371ca1fda0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:26Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-ea140-predictor-85487667c-ztpzz namespace: kserve-ci-e2e-test resourceVersion: "18993" uid: 3b1ddd12-bc61-4aff-9907-b08cd577ee3a kind: Event lastTimestamp: "2026-05-26T16:19:26Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:19:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:19:26Z" name: raw-sklearn-ea140-predictor-85487667c-ztpzz.18b329eacd43dd3e namespace: kserve-ci-e2e-test resourceVersion: "19009" uid: 4ef449e3-abe1-418f-87a3-b6210ab26b35 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-ea140-predictor-85487667c-ztpzz namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: 3b1ddd12-bc61-4aff-9907-b08cd577ee3a kind: Event lastTimestamp: "2026-05-26T16:19:26Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0413900e48a4ba69f75c45b06af88f792921f4669631dcf3a047c1fe5c1fc341" already present on machine metadata: creationTimestamp: "2026-05-26T16:19:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:19:26Z" name: raw-sklearn-ea140-predictor-85487667c-ztpzz.18b329eacfd90b67 namespace: kserve-ci-e2e-test resourceVersion: "19012" uid: ba64574b-927b-4b41-9e44-c42f78559d66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-ea140-predictor-85487667c-ztpzz namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: 3b1ddd12-bc61-4aff-9907-b08cd577ee3a kind: Event lastTimestamp: "2026-05-26T16:19:26Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T16:19:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:19:26Z" name: raw-sklearn-ea140-predictor-85487667c-ztpzz.18b329ead2c51df7 namespace: kserve-ci-e2e-test resourceVersion: "19013" uid: 1b17212e-93f1-4a90-a0be-cf476ba179f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-ea140-predictor-85487667c-ztpzz namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: 3b1ddd12-bc61-4aff-9907-b08cd577ee3a kind: Event lastTimestamp: "2026-05-26T16:19:26Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T16:19:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:19:26Z" name: raw-sklearn-ea140-predictor-85487667c-ztpzz.18b329ead32a96e7 namespace: kserve-ci-e2e-test resourceVersion: "19014" uid: 67395584-9175-4126-984f-723dae113ca6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-ea140-predictor-85487667c-ztpzz namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: 3b1ddd12-bc61-4aff-9907-b08cd577ee3a kind: Event lastTimestamp: "2026-05-26T16:19:30Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:19:30Z" name: raw-sklearn-ea140-predictor-85487667c-ztpzz.18b329ebc4d5d724 namespace: kserve-ci-e2e-test resourceVersion: "19049" uid: 44ee60bc-dc19-4d3e-bde2-9852b8847e75 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-ea140-predictor-85487667c-ztpzz namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: 3b1ddd12-bc61-4aff-9907-b08cd577ee3a kind: Event lastTimestamp: "2026-05-26T16:19:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:19:30Z" name: raw-sklearn-ea140-predictor-85487667c-ztpzz.18b329ebc81f664b namespace: kserve-ci-e2e-test resourceVersion: "19051" uid: e4f618cc-4aa0-4134-b015-78118ce8b8f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-ea140-predictor-85487667c-ztpzz namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: 3b1ddd12-bc61-4aff-9907-b08cd577ee3a kind: Event lastTimestamp: "2026-05-26T16:19:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:19:30Z" name: raw-sklearn-ea140-predictor-85487667c-ztpzz.18b329ebc8c94efe namespace: kserve-ci-e2e-test resourceVersion: "19052" uid: dcc40b25-4366-4d63-8689-848223b39ccb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-ea140-predictor-85487667c-ztpzz namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: 3b1ddd12-bc61-4aff-9907-b08cd577ee3a kind: Event lastTimestamp: "2026-05-26T16:19:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:19:30Z" name: raw-sklearn-ea140-predictor-85487667c-ztpzz.18b329ebc8d46195 namespace: kserve-ci-e2e-test resourceVersion: "19053" uid: e6cb6691-dba8-48af-a8d3-f0d15d3e4148 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-ea140-predictor-85487667c-ztpzz namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: 3b1ddd12-bc61-4aff-9907-b08cd577ee3a kind: Event lastTimestamp: "2026-05-26T16:19:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:19:30Z" name: raw-sklearn-ea140-predictor-85487667c-ztpzz.18b329ebcbeb61fa namespace: kserve-ci-e2e-test resourceVersion: "19056" uid: 2d0ccfcb-464b-4c0b-8e2b-8165af255b93 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-ea140-predictor-85487667c-ztpzz namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: 3b1ddd12-bc61-4aff-9907-b08cd577ee3a kind: Event lastTimestamp: "2026-05-26T16:19:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:19:30Z" name: raw-sklearn-ea140-predictor-85487667c-ztpzz.18b329ebcc54eb2e namespace: kserve-ci-e2e-test resourceVersion: "19057" uid: 4bc32dd2-66cb-4245-b84d-8149347a13e0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T16:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-ea140-predictor-85487667c-ztpzz namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: 3b1ddd12-bc61-4aff-9907-b08cd577ee3a kind: Event lastTimestamp: "2026-05-26T16:20:48Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:20:48Z" name: raw-sklearn-ea140-predictor-85487667c-ztpzz.18b329ec3c9bd0b0 namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: ad0dabf1-6ac0-495f-9bb2-3842d68b002e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:20:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-ea140-predictor-85487667c-ztpzz namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: 3b1ddd12-bc61-4aff-9907-b08cd577ee3a kind: Event lastTimestamp: "2026-05-26T16:20:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:20:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:20:45Z" name: raw-sklearn-ea140-predictor-85487667c-ztpzz.18b329fd314631d6 namespace: kserve-ci-e2e-test resourceVersion: "19624" uid: 6e1ea450-92ca-40c4-8184-32621a26e27c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:20:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-ea140-predictor-85487667c-ztpzz namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: 3b1ddd12-bc61-4aff-9907-b08cd577ee3a kind: Event lastTimestamp: "2026-05-26T16:20:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:20:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:20:45Z" name: raw-sklearn-ea140-predictor-85487667c-ztpzz.18b329fd31472958 namespace: kserve-ci-e2e-test resourceVersion: "19625" uid: 08f97110-ba4d-49e2-ae7f-02e169856a8e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:20:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-ea140-predictor-85487667c-ztpzz namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: 3b1ddd12-bc61-4aff-9907-b08cd577ee3a kind: Event lastTimestamp: "2026-05-26T16:20:48Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:20:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:20:48Z" name: raw-sklearn-ea140-predictor-85487667c-ztpzz.18b329fdeead3d77 namespace: kserve-ci-e2e-test resourceVersion: "19688" uid: 9897115e-0664-4f8a-a58b-5522c78dc4ff reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-ea140-predictor-85487667c namespace: kserve-ci-e2e-test resourceVersion: "18974" uid: 19f523af-1a02-4794-aa75-96d4410f2fcb kind: Event lastTimestamp: "2026-05-26T16:19:25Z" message: 'Created pod: raw-sklearn-ea140-predictor-85487667c-ztpzz' metadata: creationTimestamp: "2026-05-26T16:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:19:25Z" name: raw-sklearn-ea140-predictor-85487667c.18b329ea8f76e4d5 namespace: kserve-ci-e2e-test resourceVersion: "18986" uid: 5d2d539c-82f7-47ad-8ccc-eba3a58f1362 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-ea140-predictor namespace: kserve-ci-e2e-test resourceVersion: "18972" uid: b3de44d2-b689-49e5-9a91-41538496dd7a kind: Event lastTimestamp: "2026-05-26T16:19:25Z" message: Scaled up replica set raw-sklearn-ea140-predictor-85487667c from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:19:25Z" name: raw-sklearn-ea140-predictor.18b329ea8a334955 namespace: kserve-ci-e2e-test resourceVersion: "18976" uid: d8b48298-d800-466a-ac44-dd0e191ea2e0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:19:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-ea140-predictor namespace: kserve-ci-e2e-test resourceVersion: "18973" uid: b45cbe47-726f-4277-9795-2d56ded3ec86 kind: Event lastTimestamp: "2026-05-26T16:19:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:19:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:19:55Z" name: raw-sklearn-ea140-predictor.18b329ee0938ad58 namespace: kserve-ci-e2e-test resourceVersion: "19239" uid: 6c5acdbb-d515-491a-8f8b-cb61ce7bfe44 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:19:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-ea140-predictor namespace: kserve-ci-e2e-test resourceVersion: "18973" uid: b45cbe47-726f-4277-9795-2d56ded3ec86 kind: Event lastTimestamp: "2026-05-26T16:19:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:19:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:19:55Z" name: raw-sklearn-ea140-predictor.18b329ee09bd9ee4 namespace: kserve-ci-e2e-test resourceVersion: "19240" uid: 630f683c-a249-42de-9f12-7bd397c852de reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:20:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-ea140-predictor namespace: kserve-ci-e2e-test resourceVersion: "19129" uid: b45cbe47-726f-4277-9795-2d56ded3ec86 kind: Event lastTimestamp: "2026-05-26T16:20:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:20:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:20:40Z" name: raw-sklearn-ea140-predictor.18b329f507f7767c namespace: kserve-ci-e2e-test resourceVersion: "19559" uid: 74cc7a93-cb0d-44cc-b077-0ae946159809 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:20:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-ea140-predictor namespace: kserve-ci-e2e-test resourceVersion: "19129" uid: b45cbe47-726f-4277-9795-2d56ded3ec86 kind: Event lastTimestamp: "2026-05-26T16:20:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:20:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:20:40Z" name: raw-sklearn-ea140-predictor.18b329f5087a21a2 namespace: kserve-ci-e2e-test resourceVersion: "19560" uid: 733976a0-8864-4f88-a94b-2403ec655a28 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-05-26T16:19:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-ea140 namespace: kserve-ci-e2e-test resourceVersion: "18960" uid: 805cf733-752b-4a0e-8812-f1aee0bc7823 kind: Event lastTimestamp: "2026-05-26T16:20:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:20:38Z" name: raw-sklearn-ea140.18b329ea89dda41b namespace: kserve-ci-e2e-test resourceVersion: "19548" uid: 0098d03b-2ee6-48cf-96d3-f44d8f7303a0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:19:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-ea140 namespace: kserve-ci-e2e-test resourceVersion: "18960" uid: 805cf733-752b-4a0e-8812-f1aee0bc7823 kind: Event lastTimestamp: "2026-05-26T16:19:25Z" message: 'Failed to update status for InferenceService "raw-sklearn-ea140": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-ea140": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:19:25Z" name: raw-sklearn-ea140.18b329ea8a965f55 namespace: kserve-ci-e2e-test resourceVersion: "18978" uid: 7fc65606-980f-441b-9463-cfe8855ef06f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:20:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-ea140 namespace: kserve-ci-e2e-test resourceVersion: "19546" uid: 805cf733-752b-4a0e-8812-f1aee0bc7823 kind: Event lastTimestamp: "2026-05-26T16:20:38Z" message: InferenceService [raw-sklearn-ea140] is Ready metadata: creationTimestamp: "2026-05-26T16:20:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:20:38Z" name: raw-sklearn-ea140.18b329fba52492e0 namespace: kserve-ci-e2e-test resourceVersion: "19547" uid: 1da1bff7-2a11-4f93-8fa5-a577b18ad8a0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:20:45.282603Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg namespace: kserve-ci-e2e-test resourceVersion: "19637" uid: 05129dae-e202-4f9a-8aec-a47bd5cefa6d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg to ip-10-0-134-19.ec2.internal metadata: creationTimestamp: "2026-05-26T16:20:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:20:45Z" name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg.18b329fd37e3a698 namespace: kserve-ci-e2e-test resourceVersion: "19643" uid: 6afbc5c3-8cba-4052-9405-46f2024552f8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8b5dfd5d8-s8tjj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:20:45Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: 05129dae-e202-4f9a-8aec-a47bd5cefa6d kind: Event lastTimestamp: "2026-05-26T16:20:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-runtime-5a02e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:20:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:20:45Z" name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg.18b329fd41372084 namespace: kserve-ci-e2e-test resourceVersion: "19651" uid: 020df9cc-e21e-446e-b335-dbcea087c775 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:20:46Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg namespace: kserve-ci-e2e-test resourceVersion: "19647" uid: 05129dae-e202-4f9a-8aec-a47bd5cefa6d kind: Event lastTimestamp: "2026-05-26T16:20:46Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:20:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:20:46Z" name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg.18b329fd75ddb98a namespace: kserve-ci-e2e-test resourceVersion: "19663" uid: c15eda7b-fb0b-4257-b9c4-767dc102ad86 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:20:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: 05129dae-e202-4f9a-8aec-a47bd5cefa6d kind: Event lastTimestamp: "2026-05-26T16:20:46Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0413900e48a4ba69f75c45b06af88f792921f4669631dcf3a047c1fe5c1fc341" already present on machine metadata: creationTimestamp: "2026-05-26T16:20:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:20:46Z" name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg.18b329fd78254bca namespace: kserve-ci-e2e-test resourceVersion: "19665" uid: bfd1937e-a5d7-40ce-9662-ab28a96a158a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:20:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: 05129dae-e202-4f9a-8aec-a47bd5cefa6d kind: Event lastTimestamp: "2026-05-26T16:20:46Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T16:20:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:20:46Z" name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg.18b329fd7b5633c1 namespace: kserve-ci-e2e-test resourceVersion: "19666" uid: 9864b3ae-3b08-4fce-8ee6-b245f52c6caa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:20:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: 05129dae-e202-4f9a-8aec-a47bd5cefa6d kind: Event lastTimestamp: "2026-05-26T16:20:46Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T16:20:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:20:46Z" name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg.18b329fd7bd6b405 namespace: kserve-ci-e2e-test resourceVersion: "19667" uid: 0afbb438-64ed-42f9-bc74-c732ee2f094c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:20:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: 05129dae-e202-4f9a-8aec-a47bd5cefa6d kind: Event lastTimestamp: "2026-05-26T16:20:50Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:20:50Z" name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg.18b329fe74f6a17f namespace: kserve-ci-e2e-test resourceVersion: "19706" uid: 979bcb6a-0fd8-4a27-8d95-f246ce3d391d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:20:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: 05129dae-e202-4f9a-8aec-a47bd5cefa6d kind: Event lastTimestamp: "2026-05-26T16:20:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:20:50Z" name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg.18b329fe77d3512e namespace: kserve-ci-e2e-test resourceVersion: "19709" uid: 0295c0ec-d8d8-4c7d-bfa5-b84e60ca68b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:20:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: 05129dae-e202-4f9a-8aec-a47bd5cefa6d kind: Event lastTimestamp: "2026-05-26T16:20:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:20:50Z" name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg.18b329fe78376896 namespace: kserve-ci-e2e-test resourceVersion: "19710" uid: a534713b-c1c1-4040-bfa9-2a2802f128db reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:20:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: 05129dae-e202-4f9a-8aec-a47bd5cefa6d kind: Event lastTimestamp: "2026-05-26T16:20:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:20:50Z" name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg.18b329fe7842f546 namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: 6595c13f-af0d-4440-b5e7-ed882ee78d6c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:20:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: 05129dae-e202-4f9a-8aec-a47bd5cefa6d kind: Event lastTimestamp: "2026-05-26T16:20:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:20:50Z" name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg.18b329fe7b16b343 namespace: kserve-ci-e2e-test resourceVersion: "19714" uid: 4c571856-cf7d-432e-8de0-35547f0805ce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:20:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: 05129dae-e202-4f9a-8aec-a47bd5cefa6d kind: Event lastTimestamp: "2026-05-26T16:20:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:20:50Z" name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg.18b329fe7b81295a namespace: kserve-ci-e2e-test resourceVersion: "19715" uid: 1b157a6b-f11c-470f-8b8f-8f5c28ffaaec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T16:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: 05129dae-e202-4f9a-8aec-a47bd5cefa6d kind: Event lastTimestamp: "2026-05-26T16:22:07Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:22:07Z" name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg.18b329feb0ed5dd5 namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 2c206fc8-cdb0-4ff3-89ac-9d9f8c40337e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:22:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: 05129dae-e202-4f9a-8aec-a47bd5cefa6d kind: Event lastTimestamp: "2026-05-26T16:22:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:22:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:22:05Z" name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg.18b32a0fe084be99 namespace: kserve-ci-e2e-test resourceVersion: "20276" uid: cdb10640-00ad-4257-8dd2-c878f81fc178 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:22:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: 05129dae-e202-4f9a-8aec-a47bd5cefa6d kind: Event lastTimestamp: "2026-05-26T16:22:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:22:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:22:05Z" name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg.18b32a0fe0853ee4 namespace: kserve-ci-e2e-test resourceVersion: "20277" uid: d98eab0b-e894-4b3d-8ee0-fbce1169270d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: 05129dae-e202-4f9a-8aec-a47bd5cefa6d kind: Event lastTimestamp: "2026-05-26T16:22:07Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:22:07Z" name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg.18b32a106304dc7f namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 87ce7fff-c7a0-4f7f-814f-04af838758ea reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-19.ec2.internal source: component: kubelet host: ip-10-0-134-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:20:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7 namespace: kserve-ci-e2e-test resourceVersion: "19628" uid: 48959e67-3633-403e-8fa5-00834f4d4d4e kind: Event lastTimestamp: "2026-05-26T16:20:45Z" message: 'Created pod: raw-sklearn-runtime-5a02e-predictor-76c678f7d7-nvwcg' metadata: creationTimestamp: "2026-05-26T16:20:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:20:45Z" name: raw-sklearn-runtime-5a02e-predictor-76c678f7d7.18b329fd377ad04a namespace: kserve-ci-e2e-test resourceVersion: "19641" uid: 06ad9440-218f-4bee-84ae-601d985b48b6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:20:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-5a02e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19626" uid: afe2d44b-dd77-4a57-8871-ba9f07b95271 kind: Event lastTimestamp: "2026-05-26T16:20:45Z" message: Scaled up replica set raw-sklearn-runtime-5a02e-predictor-76c678f7d7 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:20:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:20:45Z" name: raw-sklearn-runtime-5a02e-predictor.18b329fd331df5d5 namespace: kserve-ci-e2e-test resourceVersion: "19630" uid: 88cb934b-3bbe-4cb1-a610-72d29be3fd9b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:21:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-5a02e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19627" uid: 0800f5fb-7210-45b6-bad4-a67e1a5b390e kind: Event lastTimestamp: "2026-05-26T16:21:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:21:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:21:30Z" name: raw-sklearn-runtime-5a02e-predictor.18b32a00b1ba08cf namespace: kserve-ci-e2e-test resourceVersion: "19999" uid: 889d04d4-08eb-44ee-b7d9-7ec09a58abf9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:21:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-5a02e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19627" uid: 0800f5fb-7210-45b6-bad4-a67e1a5b390e kind: Event lastTimestamp: "2026-05-26T16:21:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:21:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:21:30Z" name: raw-sklearn-runtime-5a02e-predictor.18b32a00b249b379 namespace: kserve-ci-e2e-test resourceVersion: "20001" uid: 33c4033c-a68c-41c1-aa25-48a9bc8d023b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:21:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-5a02e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19784" uid: 0800f5fb-7210-45b6-bad4-a67e1a5b390e kind: Event lastTimestamp: "2026-05-26T16:22:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:22:00Z" name: raw-sklearn-runtime-5a02e-predictor.18b32a0431b14ef9 namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: e178b2f6-f568-4d19-aa48-e87f3ee925d1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:21:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-5a02e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19784" uid: 0800f5fb-7210-45b6-bad4-a67e1a5b390e kind: Event lastTimestamp: "2026-05-26T16:22:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:22:00Z" name: raw-sklearn-runtime-5a02e-predictor.18b32a043231e684 namespace: kserve-ci-e2e-test resourceVersion: "20223" uid: d3993a92-2329-4fdc-ae4a-8f7f8166e1cd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-05-26T16:20:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-5a02e namespace: kserve-ci-e2e-test resourceVersion: "19598" uid: 8e24d984-599f-44e8-a637-3c3c6750568a kind: Event lastTimestamp: "2026-05-26T16:21:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:20:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:21:57Z" name: raw-sklearn-runtime-5a02e.18b329fd32e6e5da namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: 924579c7-accc-4d17-ba31-2738b0e8c34e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:20:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-5a02e namespace: kserve-ci-e2e-test resourceVersion: "19598" uid: 8e24d984-599f-44e8-a637-3c3c6750568a kind: Event lastTimestamp: "2026-05-26T16:20:45Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-5a02e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-5a02e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:20:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:20:45Z" name: raw-sklearn-runtime-5a02e.18b329fd33788b76 namespace: kserve-ci-e2e-test resourceVersion: "19632" uid: 7b38af2a-0ae7-4969-83f5-c52a0849b63b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:21:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-5a02e namespace: kserve-ci-e2e-test resourceVersion: "20197" uid: 8e24d984-599f-44e8-a637-3c3c6750568a kind: Event lastTimestamp: "2026-05-26T16:21:57Z" message: InferenceService [raw-sklearn-runtime-5a02e] is Ready metadata: creationTimestamp: "2026-05-26T16:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:21:57Z" name: raw-sklearn-runtime-5a02e.18b32a0e1574ee9a namespace: kserve-ci-e2e-test resourceVersion: "20199" uid: cb9c667f-3aa8-4890-8f23-d8a84df81fe9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "20499"