--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:13:01.106060Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-5052e1-predictor-68cfc88b5c-4jqgk namespace: kserve-ci-e2e-test resourceVersion: "19221" uid: 6e1c76fc-0856-4d86-8411-8e1687f408a4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-5052e1-predictor-68cfc88b5c-4jqgk to ip-10-0-132-102.ec2.internal metadata: creationTimestamp: "2026-04-23T18:13:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:13:01Z" name: isvc-init-fail-5052e1-predictor-68cfc88b5c-4jqgk.18a90ef6cb1205d6 namespace: kserve-ci-e2e-test resourceVersion: "19226" uid: 565273aa-a94b-4c84-8367-013d79cceb01 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-774bdb58b-lw5wg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-5052e1-predictor-68cfc88b5c-4jqgk namespace: kserve-ci-e2e-test resourceVersion: "19224" uid: 6e1c76fc-0856-4d86-8411-8e1687f408a4 kind: Event lastTimestamp: "2026-04-23T18:13:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-init-fail-5052e1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:13:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:01Z" name: isvc-init-fail-5052e1-predictor-68cfc88b5c-4jqgk.18a90ef6d89d252f namespace: kserve-ci-e2e-test resourceVersion: "19236" uid: 22dc9772-eaaa-4cc9-a1c2-3e90648a9bbe reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-5052e1-predictor-68cfc88b5c-4jqgk namespace: kserve-ci-e2e-test resourceVersion: "19230" uid: 6e1c76fc-0856-4d86-8411-8e1687f408a4 kind: Event lastTimestamp: "2026-04-23T18:13:02Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:13:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:13:02Z" name: isvc-init-fail-5052e1-predictor-68cfc88b5c-4jqgk.18a90ef707e24514 namespace: kserve-ci-e2e-test resourceVersion: "19248" uid: 442487a1-44a6-4639-a59c-bea340b351d2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:13:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-5052e1-predictor-68cfc88b5c-4jqgk namespace: kserve-ci-e2e-test resourceVersion: "19224" uid: 6e1c76fc-0856-4d86-8411-8e1687f408a4 kind: Event lastTimestamp: "2026-04-23T18:13:07Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:48bcc0e7da25ba7560a82df7216c0ad464b268d851c665b85b9195165f27b89d" already present on machine metadata: creationTimestamp: "2026-04-23T18:13:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:07Z" name: isvc-init-fail-5052e1-predictor-68cfc88b5c-4jqgk.18a90ef709667f20 namespace: kserve-ci-e2e-test resourceVersion: "19302" uid: 756a70f4-5960-4810-bbe2-58ed87adf2aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:13:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-5052e1-predictor-68cfc88b5c-4jqgk namespace: kserve-ci-e2e-test resourceVersion: "19224" uid: 6e1c76fc-0856-4d86-8411-8e1687f408a4 kind: Event lastTimestamp: "2026-04-23T18:13:07Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T18:13:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:07Z" name: isvc-init-fail-5052e1-predictor-68cfc88b5c-4jqgk.18a90ef70c395237 namespace: kserve-ci-e2e-test resourceVersion: "19304" uid: 64ec7b6d-8cd3-4db6-b7ab-f3279b5a4120 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:13:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-5052e1-predictor-68cfc88b5c-4jqgk namespace: kserve-ci-e2e-test resourceVersion: "19224" uid: 6e1c76fc-0856-4d86-8411-8e1687f408a4 kind: Event lastTimestamp: "2026-04-23T18:13:07Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T18:13:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:07Z" name: isvc-init-fail-5052e1-predictor-68cfc88b5c-4jqgk.18a90ef70cf43e3c namespace: kserve-ci-e2e-test resourceVersion: "19307" uid: 38e573e0-4681-4dfc-ac6e-f5ef3e55dba5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-5052e1-predictor-68cfc88b5c-4jqgk namespace: kserve-ci-e2e-test resourceVersion: "19224" uid: 6e1c76fc-0856-4d86-8411-8e1687f408a4 kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:11Z" name: isvc-init-fail-5052e1-predictor-68cfc88b5c-4jqgk.18a90ef920716768 namespace: kserve-ci-e2e-test resourceVersion: "19353" uid: d49c73cb-4267-42bf-bc35-c2a49a3d4a5d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-5052e1-predictor-68cfc88b5c namespace: kserve-ci-e2e-test resourceVersion: "19213" uid: 05028116-fa21-4b40-aed9-ce54643df840 kind: Event lastTimestamp: "2026-04-23T18:13:01Z" message: 'Created pod: isvc-init-fail-5052e1-predictor-68cfc88b5c-4jqgk' metadata: creationTimestamp: "2026-04-23T18:13:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:01Z" name: isvc-init-fail-5052e1-predictor-68cfc88b5c.18a90ef6ca8068e6 namespace: kserve-ci-e2e-test resourceVersion: "19223" uid: 181e7a32-c92f-4c7a-a59f-d24e75ef598e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-5052e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19211" uid: 991d9bf5-ce9c-4886-b6db-5fd7a8cc57a4 kind: Event lastTimestamp: "2026-04-23T18:13:01Z" message: Scaled up replica set isvc-init-fail-5052e1-predictor-68cfc88b5c from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:13:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:01Z" name: isvc-init-fail-5052e1-predictor.18a90ef6c751d5a0 namespace: kserve-ci-e2e-test resourceVersion: "19215" uid: 7e3e4602-0030-4c4e-93b1-1c6d22fb7bda reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T18:13:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-5052e1 namespace: kserve-ci-e2e-test resourceVersion: "19185" uid: bb31b210-df28-48f2-a181-8b56b9f07f45 kind: Event lastTimestamp: "2026-04-23T18:13:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:13:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:07Z" name: isvc-init-fail-5052e1.18a90ef6c7300a42 namespace: kserve-ci-e2e-test resourceVersion: "19309" uid: 227636e2-2662-4fef-bd7b-0b20f5f7b991 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-5052e1 namespace: kserve-ci-e2e-test resourceVersion: "19185" uid: bb31b210-df28-48f2-a181-8b56b9f07f45 kind: Event lastTimestamp: "2026-04-23T18:13:01Z" message: 'Failed to update status for InferenceService "isvc-init-fail-5052e1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-5052e1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:13:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:01Z" name: isvc-init-fail-5052e1.18a90ef6c7a79a4b namespace: kserve-ci-e2e-test resourceVersion: "19217" uid: d4f6a3b6-96b0-4fc5-bc6d-a216dc8c3a07 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-5052e1 namespace: kserve-ci-e2e-test resourceVersion: "19185" uid: bb31b210-df28-48f2-a181-8b56b9f07f45 kind: Event lastTimestamp: "2026-04-23T18:13:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-5052e1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:13:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:01Z" name: isvc-init-fail-5052e1.18a90ef6c7a7d92d namespace: kserve-ci-e2e-test resourceVersion: "19218" uid: 2dee4cb9-25f0-4a89-8d7a-b094d6342d77 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:07:29.268363Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p namespace: kserve-ci-e2e-test resourceVersion: "16468" uid: 72df63a8-0670-4a87-b85a-926ad14f6594 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p to ip-10-0-132-102.ec2.internal metadata: creationTimestamp: "2026-04-23T18:07:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:07:29Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p.18a90ea9880095e7 namespace: kserve-ci-e2e-test resourceVersion: "16472" uid: 84c2ea99-63f1-4e59-bbfc-eb455c0c9a5a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-774bdb58b-lw5wg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:29Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p namespace: kserve-ci-e2e-test resourceVersion: "16475" uid: 72df63a8-0670-4a87-b85a-926ad14f6594 kind: Event lastTimestamp: "2026-04-23T18:07:29Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:07:29Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p.18a90ea9a0d2d48b namespace: kserve-ci-e2e-test resourceVersion: "16479" uid: 0590019d-9e30-453c-beee-4e5a481d2f9d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 72df63a8-0670-4a87-b85a-926ad14f6594 kind: Event lastTimestamp: "2026-04-23T18:07:29Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:48bcc0e7da25ba7560a82df7216c0ad464b268d851c665b85b9195165f27b89d" already present on machine metadata: creationTimestamp: "2026-04-23T18:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:29Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p.18a90ea9a24c80c5 namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: 782de4e0-3946-4507-a3da-8133c5d46735 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 72df63a8-0670-4a87-b85a-926ad14f6594 kind: Event lastTimestamp: "2026-04-23T18:07:29Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T18:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:29Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p.18a90ea9a4ec7c5d namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 0bef06a1-17de-4cf8-b3c8-206355ede72b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 72df63a8-0670-4a87-b85a-926ad14f6594 kind: Event lastTimestamp: "2026-04-23T18:07:29Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T18:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:29Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p.18a90ea9a5583594 namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 37b9656a-29c1-405e-a56c-70fe7fd93bc5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 72df63a8-0670-4a87-b85a-926ad14f6594 kind: Event lastTimestamp: "2026-04-23T18:07:33Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:07:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:33Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p.18a90eaa9b036b0f namespace: kserve-ci-e2e-test resourceVersion: "16520" uid: 591295e6-f43f-4de0-b585-5f286e4a7b0d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 72df63a8-0670-4a87-b85a-926ad14f6594 kind: Event lastTimestamp: "2026-04-23T18:07:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:07:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:33Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p.18a90eaa9df797ea namespace: kserve-ci-e2e-test resourceVersion: "16523" uid: 0bd0a802-764b-493c-80bb-4c22188e1c99 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 72df63a8-0670-4a87-b85a-926ad14f6594 kind: Event lastTimestamp: "2026-04-23T18:07:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:07:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:33Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p.18a90eaa9e7d3eba namespace: kserve-ci-e2e-test resourceVersion: "16525" uid: 699555f2-ae52-485f-86c7-837ae5b18804 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 72df63a8-0670-4a87-b85a-926ad14f6594 kind: Event lastTimestamp: "2026-04-23T18:07:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:07:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:33Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p.18a90eaa9e893d5c namespace: kserve-ci-e2e-test resourceVersion: "16526" uid: 89af4f0b-a07d-4757-97aa-e70fc537b407 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 72df63a8-0670-4a87-b85a-926ad14f6594 kind: Event lastTimestamp: "2026-04-23T18:07:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:07:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:33Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p.18a90eaaa1525bf3 namespace: kserve-ci-e2e-test resourceVersion: "16528" uid: d9bb0429-0ded-41bc-8a33-19502e19c810 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 72df63a8-0670-4a87-b85a-926ad14f6594 kind: Event lastTimestamp: "2026-04-23T18:07:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:07:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:33Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p.18a90eaaa1bf7be3 namespace: kserve-ci-e2e-test resourceVersion: "16529" uid: 360e7c82-0c0a-4d9c-93cd-63008ab4ac0d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 72df63a8-0670-4a87-b85a-926ad14f6594 kind: Event lastTimestamp: "2026-04-23T18:07:33Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:1f655ab77a6d99cdde8a7f1f5e600af19cd3c90848f1adcb2e5105b44606fd11" already present on machine metadata: creationTimestamp: "2026-04-23T18:07:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:34Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p.18a90eaaa1cc7075 namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: 0bf188cc-268b-49bd-85bd-e25bb979a97e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 72df63a8-0670-4a87-b85a-926ad14f6594 kind: Event lastTimestamp: "2026-04-23T18:07:34Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-23T18:07:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:34Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p.18a90eaaa52ca6a0 namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: 194db714-92af-4f92-a6d6-582abc6ce15c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 72df63a8-0670-4a87-b85a-926ad14f6594 kind: Event lastTimestamp: "2026-04-23T18:07:34Z" message: Started container agent metadata: creationTimestamp: "2026-04-23T18:07:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:34Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p.18a90eaaa5a63aa7 namespace: kserve-ci-e2e-test resourceVersion: "16534" uid: 5fc1cda4-e524-43dd-acdd-d84a3704fb54 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-23T18:07:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 72df63a8-0670-4a87-b85a-926ad14f6594 kind: Event lastTimestamp: "2026-04-23T18:09:11Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:07:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:09:11Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p.18a90eaad6f05848 namespace: kserve-ci-e2e-test resourceVersion: "17367" uid: 35cf84a5-a890-4edd-bd9d-18f4f7daaef1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T18:07:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 72df63a8-0670-4a87-b85a-926ad14f6594 kind: Event lastTimestamp: "2026-04-23T18:09:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:07:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:09:11Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p.18a90eab12d26c2d namespace: kserve-ci-e2e-test resourceVersion: "17368" uid: 53733db2-0018-41ab-8140-376c9c249efe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:08:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 72df63a8-0670-4a87-b85a-926ad14f6594 kind: Event lastTimestamp: "2026-04-23T18:08:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:08:54Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p.18a90ebd5d9bc6e3 namespace: kserve-ci-e2e-test resourceVersion: "17173" uid: a6d35bf3-8abd-4617-abc7-0fae5e51b73c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:08:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 72df63a8-0670-4a87-b85a-926ad14f6594 kind: Event lastTimestamp: "2026-04-23T18:08:54Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-23T18:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:08:54Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p.18a90ebd5d9cfd18 namespace: kserve-ci-e2e-test resourceVersion: "17175" uid: 91bfc648-b28d-4daa-872f-0cfb20691ce9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:08:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 72df63a8-0670-4a87-b85a-926ad14f6594 kind: Event lastTimestamp: "2026-04-23T18:08:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:08:54Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p.18a90ebd5d9d0db4 namespace: kserve-ci-e2e-test resourceVersion: "17174" uid: eb5f7e87-fe7c-45a9-9ec1-b1862aedf1a3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:08:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 72df63a8-0670-4a87-b85a-926ad14f6594 kind: Event lastTimestamp: "2026-04-23T18:09:11Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp 10.133.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:09:11Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p.18a90ebdeef353c9 namespace: kserve-ci-e2e-test resourceVersion: "17366" uid: 7968b27a-5033-48e9-969a-40c229caeae9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-f1c6b-predictor-796f9c8994 namespace: kserve-ci-e2e-test resourceVersion: "16454" uid: bb0da67c-bfec-49cb-9d1d-c783418521f9 kind: Event lastTimestamp: "2026-04-23T18:07:29Z" message: 'Created pod: isvc-logger-raw-f1c6b-predictor-796f9c8994-2429p' metadata: creationTimestamp: "2026-04-23T18:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:07:29Z" name: isvc-logger-raw-f1c6b-predictor-796f9c8994.18a90ea9878d9596 namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: aaf84ba8-0d52-4383-ab1b-3be70769163b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-f1c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: 4f4691eb-bcbf-4139-9e31-f97b60a5dd48 kind: Event lastTimestamp: "2026-04-23T18:07:29Z" message: Scaled up replica set isvc-logger-raw-f1c6b-predictor-796f9c8994 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:07:29Z" name: isvc-logger-raw-f1c6b-predictor.18a90ea9834e0c78 namespace: kserve-ci-e2e-test resourceVersion: "16456" uid: cfdbdfe9-371e-4789-8082-bb237e4db77e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:07:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-f1c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16453" uid: 1d3c2a48-ab15-4a3f-be51-9d931fe01f70 kind: Event lastTimestamp: "2026-04-23T18:07:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:07:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:07:59Z" name: isvc-logger-raw-f1c6b-predictor.18a90ead01d5b3be namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: 3f0a9e9d-de47-4619-95c9-09d2fda0f688 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:07:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-f1c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16453" uid: 1d3c2a48-ab15-4a3f-be51-9d931fe01f70 kind: Event lastTimestamp: "2026-04-23T18:07:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:07:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:07:59Z" name: isvc-logger-raw-f1c6b-predictor.18a90ead0244880d namespace: kserve-ci-e2e-test resourceVersion: "16728" uid: 084bdd97-8aa3-4dcc-ad79-7513e9db4f3e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:08:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-f1c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16617" uid: 1d3c2a48-ab15-4a3f-be51-9d931fe01f70 kind: Event lastTimestamp: "2026-04-23T18:08:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:08:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:08:44Z" name: isvc-logger-raw-f1c6b-predictor.18a90eb4003984c7 namespace: kserve-ci-e2e-test resourceVersion: "17061" uid: 1336fb74-75db-47ad-9b89-f269b78991e0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:08:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-f1c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16617" uid: 1d3c2a48-ab15-4a3f-be51-9d931fe01f70 kind: Event lastTimestamp: "2026-04-23T18:08:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:08:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:08:44Z" name: isvc-logger-raw-f1c6b-predictor.18a90eb400ae4b13 namespace: kserve-ci-e2e-test resourceVersion: "17062" uid: d9e21fde-2935-4070-b11b-5d35a8d0593f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T18:07:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-f1c6b namespace: kserve-ci-e2e-test resourceVersion: "16441" uid: a9e8ad46-7e0b-4704-ad5d-09d770f62e8f kind: Event lastTimestamp: "2026-04-23T18:08:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:08:42Z" name: isvc-logger-raw-f1c6b.18a90ea9830c393b namespace: kserve-ci-e2e-test resourceVersion: "17045" uid: 6855b0a0-c563-43ee-85a6-4af2a3950de0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-f1c6b namespace: kserve-ci-e2e-test resourceVersion: "16441" uid: a9e8ad46-7e0b-4704-ad5d-09d770f62e8f kind: Event lastTimestamp: "2026-04-23T18:07:29Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-f1c6b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-f1c6b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:07:29Z" name: isvc-logger-raw-f1c6b.18a90ea98364fb00 namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: c3220eb6-4b3e-4454-84c7-091931d485ec reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:08:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-f1c6b namespace: kserve-ci-e2e-test resourceVersion: "17042" uid: a9e8ad46-7e0b-4704-ad5d-09d770f62e8f kind: Event lastTimestamp: "2026-04-23T18:08:42Z" message: InferenceService [isvc-logger-raw-f1c6b] is Ready metadata: creationTimestamp: "2026-04-23T18:08:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:08:42Z" name: isvc-logger-raw-f1c6b.18a90eba778c4de1 namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: 32cb1c8d-bb79-4fe5-8fe4-0b753bfbab90 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:11:24.792651Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62 namespace: kserve-ci-e2e-test resourceVersion: "18352" uid: b819f9d6-5d46-4210-96a1-25228eae230f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-b0e9d2-predictor-54c659f8-4mp62 to ip-10-0-132-102.ec2.internal metadata: creationTimestamp: "2026-04-23T18:11:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:11:24Z" name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62.18a90ee05e5805b0 namespace: kserve-ci-e2e-test resourceVersion: "18356" uid: fa1e92e9-3e49-41c9-ab3b-750706518883 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-774bdb58b-lw5wg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:24Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: b819f9d6-5d46-4210-96a1-25228eae230f kind: Event lastTimestamp: "2026-04-23T18:11:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-primary-b0e9d2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:24Z" name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62.18a90ee06982065f namespace: kserve-ci-e2e-test resourceVersion: "18365" uid: 89725dc5-b9e9-40f2-a083-632bbcede4c4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:25Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62 namespace: kserve-ci-e2e-test resourceVersion: "18361" uid: b819f9d6-5d46-4210-96a1-25228eae230f kind: Event lastTimestamp: "2026-04-23T18:11:25Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:11:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:11:25Z" name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62.18a90ee09b19398a namespace: kserve-ci-e2e-test resourceVersion: "18378" uid: 07c23db5-08eb-4a91-bd8b-a6a588025c72 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: b819f9d6-5d46-4210-96a1-25228eae230f kind: Event lastTimestamp: "2026-04-23T18:11:25Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:48bcc0e7da25ba7560a82df7216c0ad464b268d851c665b85b9195165f27b89d" already present on machine metadata: creationTimestamp: "2026-04-23T18:11:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:25Z" name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62.18a90ee09c9866ac namespace: kserve-ci-e2e-test resourceVersion: "18380" uid: daf57f1a-1aaa-4f62-adc1-d06b53b803fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: b819f9d6-5d46-4210-96a1-25228eae230f kind: Event lastTimestamp: "2026-04-23T18:11:25Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T18:11:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:25Z" name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62.18a90ee09f5f0a32 namespace: kserve-ci-e2e-test resourceVersion: "18382" uid: 0d96e529-6858-4c8b-91ce-3900def97122 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: b819f9d6-5d46-4210-96a1-25228eae230f kind: Event lastTimestamp: "2026-04-23T18:11:25Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T18:11:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:25Z" name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62.18a90ee09fd1dc93 namespace: kserve-ci-e2e-test resourceVersion: "18383" uid: c8e80c90-a8b3-4f3d-89e9-2c46a00c54bd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: b819f9d6-5d46-4210-96a1-25228eae230f kind: Event lastTimestamp: "2026-04-23T18:11:29Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:11:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:29Z" name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62.18a90ee17934bb2f namespace: kserve-ci-e2e-test resourceVersion: "18415" uid: 5ceca606-4dba-4560-b529-91fa2dd723b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: b819f9d6-5d46-4210-96a1-25228eae230f kind: Event lastTimestamp: "2026-04-23T18:11:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:11:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:29Z" name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62.18a90ee17c31b559 namespace: kserve-ci-e2e-test resourceVersion: "18420" uid: b92b669e-05c4-4c1c-815e-5eb0eed8962e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: b819f9d6-5d46-4210-96a1-25228eae230f kind: Event lastTimestamp: "2026-04-23T18:11:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:11:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:29Z" name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62.18a90ee17ca34b92 namespace: kserve-ci-e2e-test resourceVersion: "18423" uid: ff4d2c3d-81f5-4572-9d73-66e61b665e2c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: b819f9d6-5d46-4210-96a1-25228eae230f kind: Event lastTimestamp: "2026-04-23T18:11:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:11:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:29Z" name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62.18a90ee17caf9f0a namespace: kserve-ci-e2e-test resourceVersion: "18424" uid: b804b87a-ab61-4f08-94d0-5a442418852c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: b819f9d6-5d46-4210-96a1-25228eae230f kind: Event lastTimestamp: "2026-04-23T18:11:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:11:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:29Z" name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62.18a90ee17f89b9a5 namespace: kserve-ci-e2e-test resourceVersion: "18427" uid: abf391f4-3570-4777-a966-60183303bed5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: b819f9d6-5d46-4210-96a1-25228eae230f kind: Event lastTimestamp: "2026-04-23T18:11:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:11:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:29Z" name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62.18a90ee17ffc791d namespace: kserve-ci-e2e-test resourceVersion: "18428" uid: b79de68c-8a2f-420c-b488-49fa77aacf33 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-23T18:11:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: b819f9d6-5d46-4210-96a1-25228eae230f kind: Event lastTimestamp: "2026-04-23T18:12:27Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:27Z" name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62.18a90ee1f0d8421c namespace: kserve-ci-e2e-test resourceVersion: "18845" uid: 08a702af-56cc-4be5-871f-b7a007d67e4f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: b819f9d6-5d46-4210-96a1-25228eae230f kind: Event lastTimestamp: "2026-04-23T18:13:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:13:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:00Z" name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62.18a90ef6c1b37b8e namespace: kserve-ci-e2e-test resourceVersion: "19183" uid: 528dcbd5-3e1a-43fd-befd-57c88193ae79 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: b819f9d6-5d46-4210-96a1-25228eae230f kind: Event lastTimestamp: "2026-04-23T18:13:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:13:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:00Z" name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62.18a90ef6c1b4033e namespace: kserve-ci-e2e-test resourceVersion: "19184" uid: 9c003711-361d-456e-95dd-f06b20444133 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62 namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: b819f9d6-5d46-4210-96a1-25228eae230f kind: Event lastTimestamp: "2026-04-23T18:13:02Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:13:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:02Z" name: isvc-primary-b0e9d2-predictor-54c659f8-4mp62.18a90ef720fa6648 namespace: kserve-ci-e2e-test resourceVersion: "19258" uid: 039484ef-2237-4b93-9f6b-f7e21fa0b929 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-b0e9d2-predictor-54c659f8 namespace: kserve-ci-e2e-test resourceVersion: "18343" uid: e99c49bc-e2a2-4495-99ee-e4edbef5b9cb kind: Event lastTimestamp: "2026-04-23T18:11:24Z" message: 'Created pod: isvc-primary-b0e9d2-predictor-54c659f8-4mp62' metadata: creationTimestamp: "2026-04-23T18:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:11:24Z" name: isvc-primary-b0e9d2-predictor-54c659f8.18a90ee05de78c9a namespace: kserve-ci-e2e-test resourceVersion: "18353" uid: 2b0f7ec0-657d-4114-bfb1-389969f1466e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-b0e9d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18339" uid: 9495ea56-b51f-4428-9b55-3614aeb8d837 kind: Event lastTimestamp: "2026-04-23T18:11:24Z" message: Scaled up replica set isvc-primary-b0e9d2-predictor-54c659f8 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:11:24Z" name: isvc-primary-b0e9d2-predictor.18a90ee0597e61f1 namespace: kserve-ci-e2e-test resourceVersion: "18345" uid: f5279dd6-a55f-4b18-990f-2ee036230988 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:11:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-b0e9d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18342" uid: eb5d05ab-0a4c-4640-a98f-b0a87f1c8a20 kind: Event lastTimestamp: "2026-04-23T18:12:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:11:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:09Z" name: isvc-primary-b0e9d2-predictor.18a90ee3d823bf5c namespace: kserve-ci-e2e-test resourceVersion: "18717" uid: 5d987abc-b05d-4ebd-8caf-61d65d654f06 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:11:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-b0e9d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18342" uid: eb5d05ab-0a4c-4640-a98f-b0a87f1c8a20 kind: Event lastTimestamp: "2026-04-23T18:12:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:11:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:09Z" name: isvc-primary-b0e9d2-predictor.18a90ee3d89f9e3b namespace: kserve-ci-e2e-test resourceVersion: "18718" uid: 61873d2a-f6db-4c01-ad67-e7e0ecc69a54 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:12:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-b0e9d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18508" uid: eb5d05ab-0a4c-4640-a98f-b0a87f1c8a20 kind: Event lastTimestamp: "2026-04-23T18:12:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:12:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:54Z" name: isvc-primary-b0e9d2-predictor.18a90eee55db78f5 namespace: kserve-ci-e2e-test resourceVersion: "19109" uid: 0aa0de33-1612-4de0-9a23-4c5e9b517aff reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:12:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-b0e9d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18508" uid: eb5d05ab-0a4c-4640-a98f-b0a87f1c8a20 kind: Event lastTimestamp: "2026-04-23T18:12:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:12:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:54Z" name: isvc-primary-b0e9d2-predictor.18a90eee5644f6e2 namespace: kserve-ci-e2e-test resourceVersion: "19110" uid: 0b58fa79-2101-408b-8294-73338960ef58 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-23T18:11:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-b0e9d2 namespace: kserve-ci-e2e-test resourceVersion: "18313" uid: 276b6579-9733-42a4-83e1-e0c8156e665c kind: Event lastTimestamp: "2026-04-23T18:12:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:37Z" name: isvc-primary-b0e9d2.18a90ee0595ccebb namespace: kserve-ci-e2e-test resourceVersion: "18923" uid: e66e1e90-aa60-41b9-94b3-d6d63b3185da reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:11:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-b0e9d2 namespace: kserve-ci-e2e-test resourceVersion: "18313" uid: 276b6579-9733-42a4-83e1-e0c8156e665c kind: Event lastTimestamp: "2026-04-23T18:11:24Z" message: 'Failed to update status for InferenceService "isvc-primary-b0e9d2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-b0e9d2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:11:24Z" name: isvc-primary-b0e9d2.18a90ee059b6db92 namespace: kserve-ci-e2e-test resourceVersion: "18362" uid: a870ca2f-0380-4d80-821d-aa8c86c967b1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:11:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-b0e9d2 namespace: kserve-ci-e2e-test resourceVersion: "18313" uid: 276b6579-9733-42a4-83e1-e0c8156e665c kind: Event lastTimestamp: "2026-04-23T18:11:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-b0e9d2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:11:24Z" name: isvc-primary-b0e9d2.18a90ee059b71c76 namespace: kserve-ci-e2e-test resourceVersion: "18363" uid: ad715ac8-49c0-4e53-8062-c0b3cbd1ec9f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-b0e9d2 namespace: kserve-ci-e2e-test resourceVersion: "18920" uid: 276b6579-9733-42a4-83e1-e0c8156e665c kind: Event lastTimestamp: "2026-04-23T18:12:37Z" message: InferenceService [isvc-primary-b0e9d2] is Ready metadata: creationTimestamp: "2026-04-23T18:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:37Z" name: isvc-primary-b0e9d2.18a90ef1542f42d4 namespace: kserve-ci-e2e-test resourceVersion: "18922" uid: 3a1d067d-7cbb-4ff5-9eea-32763b53dcf1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:02:23.465151Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13587" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 to ip-10-0-132-102.ec2.internal metadata: creationTimestamp: "2026-04-23T18:02:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:02:23Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e6254b5de6b namespace: kserve-ci-e2e-test resourceVersion: "13592" uid: 08bfc561-7c36-47f2-9f85-96648aa8cc0d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-774bdb58b-lw5wg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:23Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13598" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:02:23Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:02:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:02:23Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e626d6e4e28 namespace: kserve-ci-e2e-test resourceVersion: "13607" uid: 2f517064-c267-468a-a512-92f349d7dff0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:02:23Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:48bcc0e7da25ba7560a82df7216c0ad464b268d851c665b85b9195165f27b89d" metadata: creationTimestamp: "2026-04-23T18:02:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:23Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e626eb17764 namespace: kserve-ci-e2e-test resourceVersion: "13609" uid: 189ce38f-1e66-4164-8d36-ee90c399694e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:02:27Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:48bcc0e7da25ba7560a82df7216c0ad464b268d851c665b85b9195165f27b89d" in 3.144s (3.144s including waiting). Image size: 301492151 bytes.' metadata: creationTimestamp: "2026-04-23T18:02:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:27Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e632a188f88 namespace: kserve-ci-e2e-test resourceVersion: "13636" uid: cae1cf92-3a9d-4b48-ab1c-eeecb8615f51 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:02:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T18:02:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:27Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e632d019eed namespace: kserve-ci-e2e-test resourceVersion: "13637" uid: de68481d-7c65-4230-b3d5-bf3db7befce9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:02:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T18:02:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:27Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e632d6fba9a namespace: kserve-ci-e2e-test resourceVersion: "13638" uid: 5209de8e-e8f2-49e3-bd9f-e7f995aef5ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:02:30Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1417" metadata: creationTimestamp: "2026-04-23T18:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:30Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e6413750016 namespace: kserve-ci-e2e-test resourceVersion: "13671" uid: a567e0a0-1c49-4dd8-9746-78c3239b797c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:02:43Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1417" in 12.616s (12.616s including waiting). Image size: 1562812853 bytes.' metadata: creationTimestamp: "2026-04-23T18:02:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:43Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e67037b9e74 namespace: kserve-ci-e2e-test resourceVersion: "13762" uid: ea5d3c7c-f826-4f8e-a10b-b3529b1e68e2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:02:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:02:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:43Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e670672e255 namespace: kserve-ci-e2e-test resourceVersion: "13763" uid: 976ae309-e5fe-40db-8dbb-87139341366f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:02:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:02:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:43Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e6706e49286 namespace: kserve-ci-e2e-test resourceVersion: "13765" uid: 4ac5f312-45d1-4fab-91b5-b24e166717d1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:02:43Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-23T18:02:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:43Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e6706f10dbc namespace: kserve-ci-e2e-test resourceVersion: "13766" uid: b7f9ff7b-85ac-4b1c-9e86-1fafb84e8642 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:02:45Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.139s (2.139s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-23T18:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:45Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e67867edbf6 namespace: kserve-ci-e2e-test resourceVersion: "13781" uid: 54d538ad-7ebe-4f6b-96d5-333c15c2e107 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:02:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:45Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e6789a5f362 namespace: kserve-ci-e2e-test resourceVersion: "13782" uid: c9a71698-6db3-46c0-bf04-699c811358ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:02:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:45Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e678a15be48 namespace: kserve-ci-e2e-test resourceVersion: "13783" uid: 3d839fd6-4b4d-4571-9f7f-074d907750f3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:02:45Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:1f655ab77a6d99cdde8a7f1f5e600af19cd3c90848f1adcb2e5105b44606fd11" metadata: creationTimestamp: "2026-04-23T18:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:45Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e678a2129d6 namespace: kserve-ci-e2e-test resourceVersion: "13784" uid: a5b65e5c-5c8e-467f-aaa3-36b6bb98c9ce reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:02:47Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:1f655ab77a6d99cdde8a7f1f5e600af19cd3c90848f1adcb2e5105b44606fd11" in 1.925s (1.925s including waiting). Image size: 238035033 bytes.' metadata: creationTimestamp: "2026-04-23T18:02:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:47Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e67fcedcc68 namespace: kserve-ci-e2e-test resourceVersion: "13800" uid: 12159aa6-ae3b-4864-861d-1e07e3fafd89 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:02:47Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-23T18:02:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:47Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e67fff851cb namespace: kserve-ci-e2e-test resourceVersion: "13803" uid: effc9f00-590f-4e00-95d1-d1e0b6b59011 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:02:47Z" message: Started container agent metadata: creationTimestamp: "2026-04-23T18:02:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:02:47Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e6800657fc1 namespace: kserve-ci-e2e-test resourceVersion: "13804" uid: 54656f38-bdcf-461e-83ef-5847cfc1eef5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T18:02:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:04:11Z" message: 'Readiness probe failed: dial tcp 10.133.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:02:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:11Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e6848bb95a1 namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: e9d378ac-0b09-4537-8f58-06ed3ae9c507 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T18:02:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:04:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:02:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:11Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e6848c4f186 namespace: kserve-ci-e2e-test resourceVersion: "14548" uid: ed31892c-fb07-4c1e-8a75-5b8972fab684 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:03:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:03:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:58Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e7875aead2d namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: aa32259a-b696-496c-9eb0-0fda9aacfeda reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:03:58Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-23T18:03:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:58Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e7875af05c8 namespace: kserve-ci-e2e-test resourceVersion: "14350" uid: 15ae73fc-4354-49e8-9411-250ad66e00cd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:03:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:03:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:58Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e7875af4ac9 namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: e30603b0-b2a3-445e-a51c-2ad71ddea483 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:04:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72 namespace: kserve-ci-e2e-test resourceVersion: "13589" uid: 555b0261-b09e-424b-a4b8-4c3ab5608adb kind: Event lastTimestamp: "2026-04-23T18:04:20Z" message: 'Readiness probe failed: Get "https://10.133.0.20:8643/healthz": dial tcp 10.133.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:04:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:20Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72.18a90e78d0cf7eb8 namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: 3c3045c3-fcd8-4c5a-9fa0-90b3af9de344 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87 namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: 581a9207-b7f2-42d5-8323-f916d01ec762 kind: Event lastTimestamp: "2026-04-23T18:02:23Z" message: 'Created pod: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87-chc72' metadata: creationTimestamp: "2026-04-23T18:02:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:23Z" name: isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87.18a90e6254510be6 namespace: kserve-ci-e2e-test resourceVersion: "13588" uid: 2b7b4707-e2db-4323-8c83-2a22a07ff24a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-3f299-predictor namespace: kserve-ci-e2e-test resourceVersion: "13576" uid: 1a6e2c7f-dd10-43e0-beca-3e012b994e01 kind: Event lastTimestamp: "2026-04-23T18:02:23Z" message: Scaled up replica set isvc-raw-sklearn-batcher-3f299-predictor-5d54bcdf87 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:02:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:02:23Z" name: isvc-raw-sklearn-batcher-3f299-predictor.18a90e624f047312 namespace: kserve-ci-e2e-test resourceVersion: "13578" uid: 50aa89a7-8295-4fc7-b8a5-89d71c56147c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:02:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-3f299-predictor namespace: kserve-ci-e2e-test resourceVersion: "13591" uid: 67ec541c-846b-4794-b464-6f96134b93ef kind: Event lastTimestamp: "2026-04-23T18:03:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:23Z" name: isvc-raw-sklearn-batcher-3f299-predictor.18a90e65d476b551 namespace: kserve-ci-e2e-test resourceVersion: "14063" uid: d5d3e761-5df2-46fd-89e1-7a13bc62fba4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:02:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-3f299-predictor namespace: kserve-ci-e2e-test resourceVersion: "13591" uid: 67ec541c-846b-4794-b464-6f96134b93ef kind: Event lastTimestamp: "2026-04-23T18:03:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:23Z" name: isvc-raw-sklearn-batcher-3f299-predictor.18a90e65d4e73dfc namespace: kserve-ci-e2e-test resourceVersion: "14064" uid: ffad5750-e1ff-4238-817f-64c6bbab3f1b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:03:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-3f299-predictor namespace: kserve-ci-e2e-test resourceVersion: "13732" uid: 67ec541c-846b-4794-b464-6f96134b93ef kind: Event lastTimestamp: "2026-04-23T18:03:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:03:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:53Z" name: isvc-raw-sklearn-batcher-3f299-predictor.18a90e73d10c1f4a namespace: kserve-ci-e2e-test resourceVersion: "14284" uid: a29c593c-2d0d-4317-9e32-2166b5df9402 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:03:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-3f299-predictor namespace: kserve-ci-e2e-test resourceVersion: "13732" uid: 67ec541c-846b-4794-b464-6f96134b93ef kind: Event lastTimestamp: "2026-04-23T18:03:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:03:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:53Z" name: isvc-raw-sklearn-batcher-3f299-predictor.18a90e73d1742985 namespace: kserve-ci-e2e-test resourceVersion: "14285" uid: 47c6f735-96c0-4211-b3c9-e7a30907ef22 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-23T18:02:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-3f299 namespace: kserve-ci-e2e-test resourceVersion: "13564" uid: 11453321-dee4-44ba-9260-5e21ec0c2905 kind: Event lastTimestamp: "2026-04-23T18:03:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:02:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:03:51Z" name: isvc-raw-sklearn-batcher-3f299.18a90e6254f24c60 namespace: kserve-ci-e2e-test resourceVersion: "14269" uid: 32e4e47a-fe03-4050-9452-90938b9af964 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-3f299 namespace: kserve-ci-e2e-test resourceVersion: "13564" uid: 11453321-dee4-44ba-9260-5e21ec0c2905 kind: Event lastTimestamp: "2026-04-23T18:02:23Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-3f299": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-3f299": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:02:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:02:23Z" name: isvc-raw-sklearn-batcher-3f299.18a90e6255991fac namespace: kserve-ci-e2e-test resourceVersion: "13595" uid: a6b3ad56-2aef-4cfa-8535-2404db582078 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:02:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-3f299 namespace: kserve-ci-e2e-test resourceVersion: "13564" uid: 11453321-dee4-44ba-9260-5e21ec0c2905 kind: Event lastTimestamp: "2026-04-23T18:02:23Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-3f299": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:02:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:02:23Z" name: isvc-raw-sklearn-batcher-3f299.18a90e6255996ae6 namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: 59b970ef-27a5-497a-92e9-90ab5d1fd9d9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-3f299 namespace: kserve-ci-e2e-test resourceVersion: "14267" uid: 11453321-dee4-44ba-9260-5e21ec0c2905 kind: Event lastTimestamp: "2026-04-23T18:03:51Z" message: InferenceService [isvc-raw-sklearn-batcher-3f299] is Ready metadata: creationTimestamp: "2026-04-23T18:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:03:51Z" name: isvc-raw-sklearn-batcher-3f299.18a90e76bd44958e namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: a89276be-2677-4ef5-8862-888ab7e429dd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:12:44.925427Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-b0e9d2-predictor-6db877c85c-8v7gd namespace: kserve-ci-e2e-test resourceVersion: "19004" uid: 669c3171-43f8-45d2-93aa-6fca7b7db984 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-b0e9d2-predictor-6db877c85c-8v7gd to ip-10-0-132-102.ec2.internal metadata: creationTimestamp: "2026-04-23T18:12:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:12:44Z" name: isvc-secondary-b0e9d2-predictor-6db877c85c-8v7gd.18a90ef306a124dd namespace: kserve-ci-e2e-test resourceVersion: "19008" uid: 98d5a19f-7141-437c-a705-44978fe5ad77 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-774bdb58b-lw5wg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-b0e9d2-predictor-6db877c85c-8v7gd namespace: kserve-ci-e2e-test resourceVersion: "19012" uid: 669c3171-43f8-45d2-93aa-6fca7b7db984 kind: Event lastTimestamp: "2026-04-23T18:12:45Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:12:45Z" name: isvc-secondary-b0e9d2-predictor-6db877c85c-8v7gd.18a90ef31feb0c39 namespace: kserve-ci-e2e-test resourceVersion: "19018" uid: bbd938ed-7257-4c60-a658-a37f032189ba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-b0e9d2-predictor-6db877c85c-8v7gd namespace: kserve-ci-e2e-test resourceVersion: "19006" uid: 669c3171-43f8-45d2-93aa-6fca7b7db984 kind: Event lastTimestamp: "2026-04-23T18:12:48Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:48bcc0e7da25ba7560a82df7216c0ad464b268d851c665b85b9195165f27b89d" already present on machine metadata: creationTimestamp: "2026-04-23T18:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:48Z" name: isvc-secondary-b0e9d2-predictor-6db877c85c-8v7gd.18a90ef32165629e namespace: kserve-ci-e2e-test resourceVersion: "19055" uid: 7a7b8be3-200b-4a2c-a83d-0205a68738cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-b0e9d2-predictor-6db877c85c-8v7gd namespace: kserve-ci-e2e-test resourceVersion: "19006" uid: 669c3171-43f8-45d2-93aa-6fca7b7db984 kind: Event lastTimestamp: "2026-04-23T18:12:48Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T18:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:48Z" name: isvc-secondary-b0e9d2-predictor-6db877c85c-8v7gd.18a90ef324189c99 namespace: kserve-ci-e2e-test resourceVersion: "19060" uid: 0177d3c1-598d-48e0-bb3e-9b51227c2207 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-b0e9d2-predictor-6db877c85c-8v7gd namespace: kserve-ci-e2e-test resourceVersion: "19006" uid: 669c3171-43f8-45d2-93aa-6fca7b7db984 kind: Event lastTimestamp: "2026-04-23T18:12:48Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T18:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:48Z" name: isvc-secondary-b0e9d2-predictor-6db877c85c-8v7gd.18a90ef3248f5e75 namespace: kserve-ci-e2e-test resourceVersion: "19061" uid: 98e82e94-448d-4221-a262-bb8367b1ebe4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-b0e9d2-predictor-6db877c85c-8v7gd namespace: kserve-ci-e2e-test resourceVersion: "19006" uid: 669c3171-43f8-45d2-93aa-6fca7b7db984 kind: Event lastTimestamp: "2026-04-23T18:12:53Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-b0e9d2-predictor-6db877c85c-8v7gd_kserve-ci-e2e-test(669c3171-43f8-45d2-93aa-6fca7b7db984) metadata: creationTimestamp: "2026-04-23T18:12:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:53Z" name: isvc-secondary-b0e9d2-predictor-6db877c85c-8v7gd.18a90ef5177e3543 namespace: kserve-ci-e2e-test resourceVersion: "19095" uid: 51070bdc-2d7f-4d79-8054-fe94390cd7c2 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-b0e9d2-predictor-6db877c85c namespace: kserve-ci-e2e-test resourceVersion: "18995" uid: 10e6d75f-8d7b-4257-9550-38ee2e239c18 kind: Event lastTimestamp: "2026-04-23T18:12:44Z" message: 'Created pod: isvc-secondary-b0e9d2-predictor-6db877c85c-8v7gd' metadata: creationTimestamp: "2026-04-23T18:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:44Z" name: isvc-secondary-b0e9d2-predictor-6db877c85c.18a90ef3061a0249 namespace: kserve-ci-e2e-test resourceVersion: "19005" uid: 45f2bfbe-fbb7-4d40-a469-9fa9807c3d0f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-b0e9d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18993" uid: de7286cc-4ae7-4d17-9bb5-849f9e547db5 kind: Event lastTimestamp: "2026-04-23T18:12:44Z" message: Scaled up replica set isvc-secondary-b0e9d2-predictor-6db877c85c from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:44Z" name: isvc-secondary-b0e9d2-predictor.18a90ef30295bf5a namespace: kserve-ci-e2e-test resourceVersion: "18997" uid: a603c688-82e9-464d-a884-67fd823c7735 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-b0e9d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18994" uid: c4fc7f55-0a04-4a20-b440-f292f70be205 kind: Event lastTimestamp: "2026-04-23T18:12:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:12:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:59Z" name: isvc-secondary-b0e9d2-predictor.18a90ef68142876f namespace: kserve-ci-e2e-test resourceVersion: "19146" uid: c59519c3-e48a-48cf-b46a-8c35df18f042 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-b0e9d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18994" uid: c4fc7f55-0a04-4a20-b440-f292f70be205 kind: Event lastTimestamp: "2026-04-23T18:12:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:12:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:59Z" name: isvc-secondary-b0e9d2-predictor.18a90ef681aba49a namespace: kserve-ci-e2e-test resourceVersion: "19148" uid: 94013ace-5a06-4d39-9e91-5c1c02aa101c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-23T18:12:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-b0e9d2 namespace: kserve-ci-e2e-test resourceVersion: "18982" uid: 0716e561-34f8-4839-9e06-c037ff747a16 kind: Event lastTimestamp: "2026-04-23T18:12:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:55Z" name: isvc-secondary-b0e9d2.18a90ef30281bfff namespace: kserve-ci-e2e-test resourceVersion: "19113" uid: b546d269-2e5d-43f3-acdd-878aab3d14aa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-b0e9d2 namespace: kserve-ci-e2e-test resourceVersion: "18982" uid: 0716e561-34f8-4839-9e06-c037ff747a16 kind: Event lastTimestamp: "2026-04-23T18:12:44Z" message: 'Failed to update status for InferenceService "isvc-secondary-b0e9d2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-b0e9d2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:44Z" name: isvc-secondary-b0e9d2.18a90ef3031ad86b namespace: kserve-ci-e2e-test resourceVersion: "18999" uid: fbd1fd88-8ae0-4400-b2f5-f1b5fd211022 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-b0e9d2 namespace: kserve-ci-e2e-test resourceVersion: "18982" uid: 0716e561-34f8-4839-9e06-c037ff747a16 kind: Event lastTimestamp: "2026-04-23T18:12:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-b0e9d2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:44Z" name: isvc-secondary-b0e9d2.18a90ef3031b2b9e namespace: kserve-ci-e2e-test resourceVersion: "19000" uid: e4575d79-e13d-4174-ae95-204da3797fcd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:03:58.627274Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9 namespace: kserve-ci-e2e-test resourceVersion: "14370" uid: 972b4d2b-3e63-4c82-b97c-cd3ac27b7bbe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9 to ip-10-0-132-102.ec2.internal metadata: creationTimestamp: "2026-04-23T18:03:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:03:58Z" name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9.18a90e787cd094d3 namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: f226ff93-0da9-4e19-a8d7-c4c582ad2d27 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-774bdb58b-lw5wg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9 namespace: kserve-ci-e2e-test resourceVersion: "14387" uid: 972b4d2b-3e63-4c82-b97c-cd3ac27b7bbe kind: Event lastTimestamp: "2026-04-23T18:03:59Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:03:59Z" name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9.18a90e7895c54fab namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: 45b7a255-f107-473e-b264-f2dd7dcdc208 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9 namespace: kserve-ci-e2e-test resourceVersion: "14373" uid: 972b4d2b-3e63-4c82-b97c-cd3ac27b7bbe kind: Event lastTimestamp: "2026-04-23T18:03:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:48bcc0e7da25ba7560a82df7216c0ad464b268d851c665b85b9195165f27b89d" already present on machine metadata: creationTimestamp: "2026-04-23T18:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:59Z" name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9.18a90e789728d916 namespace: kserve-ci-e2e-test resourceVersion: "14420" uid: 7646eaab-cf5c-4126-bbb7-0e411a72632a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9 namespace: kserve-ci-e2e-test resourceVersion: "14373" uid: 972b4d2b-3e63-4c82-b97c-cd3ac27b7bbe kind: Event lastTimestamp: "2026-04-23T18:03:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T18:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:59Z" name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9.18a90e789a5ff10d namespace: kserve-ci-e2e-test resourceVersion: "14421" uid: 52658a14-3f1a-4b47-8b9b-48225a0e46de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9 namespace: kserve-ci-e2e-test resourceVersion: "14373" uid: 972b4d2b-3e63-4c82-b97c-cd3ac27b7bbe kind: Event lastTimestamp: "2026-04-23T18:03:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T18:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:59Z" name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9.18a90e789ae528c9 namespace: kserve-ci-e2e-test resourceVersion: "14422" uid: 22c4eb5c-bcd4-4777-be68-ffc62ca7793c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9 namespace: kserve-ci-e2e-test resourceVersion: "14373" uid: 972b4d2b-3e63-4c82-b97c-cd3ac27b7bbe kind: Event lastTimestamp: "2026-04-23T18:04:03Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:04:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:03Z" name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9.18a90e799049910b namespace: kserve-ci-e2e-test resourceVersion: "14469" uid: 798c44a2-a5b6-4226-86c7-67e5983d3bf0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9 namespace: kserve-ci-e2e-test resourceVersion: "14373" uid: 972b4d2b-3e63-4c82-b97c-cd3ac27b7bbe kind: Event lastTimestamp: "2026-04-23T18:04:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:04:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:03Z" name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9.18a90e79938a8627 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: f4fc9785-28da-4b32-bc31-a1a7c1903ab7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9 namespace: kserve-ci-e2e-test resourceVersion: "14373" uid: 972b4d2b-3e63-4c82-b97c-cd3ac27b7bbe kind: Event lastTimestamp: "2026-04-23T18:04:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:04:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:03Z" name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9.18a90e7994025891 namespace: kserve-ci-e2e-test resourceVersion: "14474" uid: d46d7152-7172-4626-86ae-e779fe1a0bd0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9 namespace: kserve-ci-e2e-test resourceVersion: "14373" uid: 972b4d2b-3e63-4c82-b97c-cd3ac27b7bbe kind: Event lastTimestamp: "2026-04-23T18:04:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:04:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:03Z" name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9.18a90e799410a993 namespace: kserve-ci-e2e-test resourceVersion: "14475" uid: 30a40fe2-3488-4c41-b09f-184208c7fd1b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9 namespace: kserve-ci-e2e-test resourceVersion: "14373" uid: 972b4d2b-3e63-4c82-b97c-cd3ac27b7bbe kind: Event lastTimestamp: "2026-04-23T18:04:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:04:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:03Z" name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9.18a90e79990ce3c2 namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 8972fbee-962b-414f-8dcc-a7967bb1a5fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9 namespace: kserve-ci-e2e-test resourceVersion: "14373" uid: 972b4d2b-3e63-4c82-b97c-cd3ac27b7bbe kind: Event lastTimestamp: "2026-04-23T18:04:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:04:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:03Z" name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9.18a90e79997ff0da namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: 1a2ffbdc-4d42-4827-bd55-9c582244bf01 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T18:04:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9 namespace: kserve-ci-e2e-test resourceVersion: "14373" uid: 972b4d2b-3e63-4c82-b97c-cd3ac27b7bbe kind: Event lastTimestamp: "2026-04-23T18:05:40Z" message: 'Readiness probe failed: dial tcp 10.133.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:04:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:40Z" name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9.18a90e79cc61aff8 namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: 337cc14a-0e14-474c-898e-888325958e28 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9 namespace: kserve-ci-e2e-test resourceVersion: "14373" uid: 972b4d2b-3e63-4c82-b97c-cd3ac27b7bbe kind: Event lastTimestamp: "2026-04-23T18:05:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:38Z" name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9.18a90e8fd2efe41e namespace: kserve-ci-e2e-test resourceVersion: "15316" uid: d81c1647-d2d3-4867-80bb-248a38c274cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9 namespace: kserve-ci-e2e-test resourceVersion: "14373" uid: 972b4d2b-3e63-4c82-b97c-cd3ac27b7bbe kind: Event lastTimestamp: "2026-04-23T18:05:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:38Z" name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9.18a90e8fd2f0bd89 namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: aa74b587-dc30-4b04-9620-c1b9099a73dc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9 namespace: kserve-ci-e2e-test resourceVersion: "14373" uid: 972b4d2b-3e63-4c82-b97c-cd3ac27b7bbe kind: Event lastTimestamp: "2026-04-23T18:05:40Z" message: 'Readiness probe failed: Get "https://10.133.0.21:8643/healthz": dial tcp 10.133.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:40Z" name: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9.18a90e9026a69a87 namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: ba000e53-dbbe-4064-9a52-e67e89457bd7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-73769-predictor-57df48d995 namespace: kserve-ci-e2e-test resourceVersion: "14357" uid: 8649657b-7ad7-4379-99c0-dd5fd5640d47 kind: Event lastTimestamp: "2026-04-23T18:03:58Z" message: 'Created pod: isvc-sklearn-graph-raw-73769-predictor-57df48d995-j4hj9' metadata: creationTimestamp: "2026-04-23T18:03:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:58Z" name: isvc-sklearn-graph-raw-73769-predictor-57df48d995.18a90e787c4c04c8 namespace: kserve-ci-e2e-test resourceVersion: "14374" uid: a86d12d7-ebbd-4f12-8a06-f94083b79c62 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-73769-predictor namespace: kserve-ci-e2e-test resourceVersion: "14355" uid: 80a949b7-1452-489c-8d7d-9c67f44b2e44 kind: Event lastTimestamp: "2026-04-23T18:03:58Z" message: Scaled up replica set isvc-sklearn-graph-raw-73769-predictor-57df48d995 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:03:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:58Z" name: isvc-sklearn-graph-raw-73769-predictor.18a90e7877b285a2 namespace: kserve-ci-e2e-test resourceVersion: "14359" uid: acb8827c-c841-4678-a08f-af11524751ad reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-73769-predictor namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 3496ac93-4910-4660-9f21-8ac3a36bc105 kind: Event lastTimestamp: "2026-04-23T18:04:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:04:13Z" name: isvc-sklearn-graph-raw-73769-predictor.18a90e7bf6383024 namespace: kserve-ci-e2e-test resourceVersion: "14564" uid: 76238f5c-0455-47af-b37a-f89f790f4d5c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-73769-predictor namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 3496ac93-4910-4660-9f21-8ac3a36bc105 kind: Event lastTimestamp: "2026-04-23T18:04:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:04:13Z" name: isvc-sklearn-graph-raw-73769-predictor.18a90e7bf6a291c5 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: a1c67df3-48c7-4135-a8f7-dc3c6bd0064d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:04:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-73769-predictor namespace: kserve-ci-e2e-test resourceVersion: "14565" uid: 3496ac93-4910-4660-9f21-8ac3a36bc105 kind: Event lastTimestamp: "2026-04-23T18:05:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:04:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:05:28Z" name: isvc-sklearn-graph-raw-73769-predictor.18a90e7f75ccceec namespace: kserve-ci-e2e-test resourceVersion: "15166" uid: ab813b21-6e0d-4670-9d5b-559e222f0fb1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:04:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-73769-predictor namespace: kserve-ci-e2e-test resourceVersion: "14565" uid: 3496ac93-4910-4660-9f21-8ac3a36bc105 kind: Event lastTimestamp: "2026-04-23T18:05:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:04:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:05:28Z" name: isvc-sklearn-graph-raw-73769-predictor.18a90e7f763fecce namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 3c24ace3-877e-4c97-ba26-7ecd860d5cd9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T18:03:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-73769 namespace: kserve-ci-e2e-test resourceVersion: "14326" uid: 292deb53-b3f4-499c-9e09-23d604d2f42d kind: Event lastTimestamp: "2026-04-23T18:05:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:03:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:05:10Z" name: isvc-sklearn-graph-raw-73769.18a90e787789fdbc namespace: kserve-ci-e2e-test resourceVersion: "15007" uid: ca1163f9-b764-4713-99a4-579b84a89e96 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-73769 namespace: kserve-ci-e2e-test resourceVersion: "14326" uid: 292deb53-b3f4-499c-9e09-23d604d2f42d kind: Event lastTimestamp: "2026-04-23T18:03:58Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-73769": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-73769": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:03:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:03:58Z" name: isvc-sklearn-graph-raw-73769.18a90e7877e12731 namespace: kserve-ci-e2e-test resourceVersion: "14361" uid: 5eef8386-195e-4190-b4c7-a2b6ae1f5a1f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-73769 namespace: kserve-ci-e2e-test resourceVersion: "15005" uid: 292deb53-b3f4-499c-9e09-23d604d2f42d kind: Event lastTimestamp: "2026-04-23T18:05:10Z" message: InferenceService [isvc-sklearn-graph-raw-73769] is Ready metadata: creationTimestamp: "2026-04-23T18:05:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:05:10Z" name: isvc-sklearn-graph-raw-73769.18a90e89301fa783 namespace: kserve-ci-e2e-test resourceVersion: "15006" uid: 23925590-afca-4439-bb59-f16fbc39a627 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:05:38.902487Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt namespace: kserve-ci-e2e-test resourceVersion: "15322" uid: 38a8f284-3466-4635-8eed-34e77cafee06 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt to ip-10-0-132-102.ec2.internal metadata: creationTimestamp: "2026-04-23T18:05:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:05:38Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt.18a90e8fd5aef16b namespace: kserve-ci-e2e-test resourceVersion: "15330" uid: f63824dc-1b01-4d43-b6b5-9beba8476360 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-774bdb58b-lw5wg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 38a8f284-3466-4635-8eed-34e77cafee06 kind: Event lastTimestamp: "2026-04-23T18:05:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-hpa-050b4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:39Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt.18a90e8fe2107b9a namespace: kserve-ci-e2e-test resourceVersion: "15380" uid: 62fb3d16-b52f-4b66-b579-7ab7d62760bc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: 38a8f284-3466-4635-8eed-34e77cafee06 kind: Event lastTimestamp: "2026-04-23T18:05:39Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:05:39Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt.18a90e9013a39f59 namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: b63a4670-f5c3-4acb-a296-dd05ec626282 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 38a8f284-3466-4635-8eed-34e77cafee06 kind: Event lastTimestamp: "2026-04-23T18:05:39Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:48bcc0e7da25ba7560a82df7216c0ad464b268d851c665b85b9195165f27b89d" already present on machine metadata: creationTimestamp: "2026-04-23T18:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:39Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt.18a90e901507d5ba namespace: kserve-ci-e2e-test resourceVersion: "15398" uid: 3458d9ac-3808-422b-ba66-0c286c04cfd8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 38a8f284-3466-4635-8eed-34e77cafee06 kind: Event lastTimestamp: "2026-04-23T18:05:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T18:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:40Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt.18a90e90180646de namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: 841c76d1-f1bf-4ec0-b245-9f30f4b055f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 38a8f284-3466-4635-8eed-34e77cafee06 kind: Event lastTimestamp: "2026-04-23T18:05:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T18:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:40Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt.18a90e90186bf311 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 6d5a59bb-6172-4cc2-b976-aa0d92481600 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 38a8f284-3466-4635-8eed-34e77cafee06 kind: Event lastTimestamp: "2026-04-23T18:05:43Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:43Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt.18a90e90ea772674 namespace: kserve-ci-e2e-test resourceVersion: "15448" uid: 1d1f1295-5c23-4380-ab1a-8b26d000532d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 38a8f284-3466-4635-8eed-34e77cafee06 kind: Event lastTimestamp: "2026-04-23T18:05:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:43Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt.18a90e90ed7068de namespace: kserve-ci-e2e-test resourceVersion: "15454" uid: 9e7afcc8-fa30-4884-a538-c22646ad00fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 38a8f284-3466-4635-8eed-34e77cafee06 kind: Event lastTimestamp: "2026-04-23T18:05:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:43Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt.18a90e90ee2d46d0 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 86c78e95-3a80-4be0-bf78-c9db95e0d977 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 38a8f284-3466-4635-8eed-34e77cafee06 kind: Event lastTimestamp: "2026-04-23T18:05:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:43Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt.18a90e90ee38a059 namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: a91e8e13-a541-4c19-baa2-5441609e5aa8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 38a8f284-3466-4635-8eed-34e77cafee06 kind: Event lastTimestamp: "2026-04-23T18:05:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:43Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt.18a90e90f1683459 namespace: kserve-ci-e2e-test resourceVersion: "15468" uid: 2eb2c021-5a09-4b17-b339-da4517c21478 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 38a8f284-3466-4635-8eed-34e77cafee06 kind: Event lastTimestamp: "2026-04-23T18:05:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:43Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt.18a90e90f1dfd852 namespace: kserve-ci-e2e-test resourceVersion: "15470" uid: 6a2eefa7-2a74-43ce-ad3b-256cba63d033 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T18:05:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 38a8f284-3466-4635-8eed-34e77cafee06 kind: Event lastTimestamp: "2026-04-23T18:07:21Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:05:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:21Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt.18a90e9162bdf18f namespace: kserve-ci-e2e-test resourceVersion: "16370" uid: 9f888f5e-145f-46d2-bde0-75be1447c78e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 38a8f284-3466-4635-8eed-34e77cafee06 kind: Event lastTimestamp: "2026-04-23T18:07:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:19Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt.18a90ea72db390ea namespace: kserve-ci-e2e-test resourceVersion: "16298" uid: 43fd3735-d66a-446f-8a30-746a4ccacd3b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 38a8f284-3466-4635-8eed-34e77cafee06 kind: Event lastTimestamp: "2026-04-23T18:07:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:19Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt.18a90ea72db4575f namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 54f3fbcb-fbd7-406c-8f29-237159ad6f69 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 38a8f284-3466-4635-8eed-34e77cafee06 kind: Event lastTimestamp: "2026-04-23T18:07:21Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:21Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt.18a90ea7bcd9306f namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: 678f1799-11be-4a6c-ac1b-0a7c8f0ed930 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559 namespace: kserve-ci-e2e-test resourceVersion: "15299" uid: 16197ce6-9c3a-49d8-abf0-e8ab05c98fa9 kind: Event lastTimestamp: "2026-04-23T18:05:38Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559-2rnkt' metadata: creationTimestamp: "2026-04-23T18:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:05:38Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559.18a90e8fd554c167 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 9d2993e9-5209-4675-b193-71ac46816837 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-050b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15293" uid: 836fb068-d2e2-466f-9230-433903cd2eb6 kind: Event lastTimestamp: "2026-04-23T18:05:38Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-050b4-predictor-55b9478559 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:05:38Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor.18a90e8fd1135bda namespace: kserve-ci-e2e-test resourceVersion: "15303" uid: 079b8da3-391f-4511-87b7-000e766d4e86 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:05:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-050b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 05e07843-2388-4a12-9e76-7a6d607c43e7 kind: Event lastTimestamp: "2026-04-23T18:06:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:05:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:06:23Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor.18a90e934ff58b09 namespace: kserve-ci-e2e-test resourceVersion: "15782" uid: 24c5cf81-619b-414d-83e3-370fdfc7153d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:05:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-050b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 05e07843-2388-4a12-9e76-7a6d607c43e7 kind: Event lastTimestamp: "2026-04-23T18:06:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:05:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:06:23Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor.18a90e93505cb9f5 namespace: kserve-ci-e2e-test resourceVersion: "15783" uid: 81c5dfb5-ed46-4e87-9be3-9d9b50535611 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:06:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-050b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15552" uid: 05e07843-2388-4a12-9e76-7a6d607c43e7 kind: Event lastTimestamp: "2026-04-23T18:07:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:07:08Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor.18a90e9dcd2edf51 namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 221693c2-61fc-42ca-901b-57c427e76ea0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:06:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-050b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15552" uid: 05e07843-2388-4a12-9e76-7a6d607c43e7 kind: Event lastTimestamp: "2026-04-23T18:07:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:07:08Z" name: isvc-sklearn-graph-raw-hpa-050b4-predictor.18a90e9dcd999958 namespace: kserve-ci-e2e-test resourceVersion: "16142" uid: 64420450-99a9-449d-a608-0264354e6b90 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T18:05:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-050b4 namespace: kserve-ci-e2e-test resourceVersion: "15280" uid: 4e8edce1-a917-4e98-bafc-c4c6fb55f625 kind: Event lastTimestamp: "2026-04-23T18:06:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:06:51Z" name: isvc-sklearn-graph-raw-hpa-050b4.18a90e8fd0f2adca namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: 16071f0b-deb0-459c-831e-dd02839c1083 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-050b4 namespace: kserve-ci-e2e-test resourceVersion: "15280" uid: 4e8edce1-a917-4e98-bafc-c4c6fb55f625 kind: Event lastTimestamp: "2026-04-23T18:05:38Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-050b4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-050b4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:05:38Z" name: isvc-sklearn-graph-raw-hpa-050b4.18a90e8fd1c8fa26 namespace: kserve-ci-e2e-test resourceVersion: "15309" uid: 7b79a1c8-4aba-4698-9c93-c5740b620d15 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:06:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-050b4 namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 4e8edce1-a917-4e98-bafc-c4c6fb55f625 kind: Event lastTimestamp: "2026-04-23T18:06:51Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-050b4] is Ready metadata: creationTimestamp: "2026-04-23T18:06:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:06:51Z" name: isvc-sklearn-graph-raw-hpa-050b4.18a90ea0c5d7715c namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: 3e7db56d-2cdd-4061-a236-8d2618bb5d26 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:08:54.483289Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m namespace: kserve-ci-e2e-test resourceVersion: "17176" uid: 405f9c55-cdf2-4708-a0cf-3c8d34ebbf7c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m to ip-10-0-132-102.ec2.internal metadata: creationTimestamp: "2026-04-23T18:08:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:08:54Z" name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m.18a90ebd5f352b35 namespace: kserve-ci-e2e-test resourceVersion: "17182" uid: 1839cee0-9b29-4f08-9cfa-de9c2796c48e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-774bdb58b-lw5wg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:08:54Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: 405f9c55-cdf2-4708-a0cf-3c8d34ebbf7c kind: Event lastTimestamp: "2026-04-23T18:08:54Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:08:54Z" name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m.18a90ebd77e557ca namespace: kserve-ci-e2e-test resourceVersion: "17209" uid: 45d02ef5-34e7-4a76-87c3-0e17ddb3caeb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:08:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: 405f9c55-cdf2-4708-a0cf-3c8d34ebbf7c kind: Event lastTimestamp: "2026-04-23T18:08:54Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:48bcc0e7da25ba7560a82df7216c0ad464b268d851c665b85b9195165f27b89d" already present on machine metadata: creationTimestamp: "2026-04-23T18:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:08:54Z" name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m.18a90ebd794ed658 namespace: kserve-ci-e2e-test resourceVersion: "17211" uid: 05a56bfc-2078-4b2b-b616-fa2e106492e7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:08:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: 405f9c55-cdf2-4708-a0cf-3c8d34ebbf7c kind: Event lastTimestamp: "2026-04-23T18:08:54Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T18:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:08:54Z" name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m.18a90ebd7c84aa03 namespace: kserve-ci-e2e-test resourceVersion: "17213" uid: 65d6a595-0423-4688-bb87-d82efd5a0c71 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:08:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: 405f9c55-cdf2-4708-a0cf-3c8d34ebbf7c kind: Event lastTimestamp: "2026-04-23T18:08:54Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T18:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:08:54Z" name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m.18a90ebd7d31a8ba namespace: kserve-ci-e2e-test resourceVersion: "17214" uid: 2b42cd15-7861-4b9c-a4a4-3dfc24c64372 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:08:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: 405f9c55-cdf2-4708-a0cf-3c8d34ebbf7c kind: Event lastTimestamp: "2026-04-23T18:08:59Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:08:59Z" name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m.18a90ebe73d5e92b namespace: kserve-ci-e2e-test resourceVersion: "17254" uid: 34aacc66-6ca2-4138-a7ce-1023eab60498 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:08:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: 405f9c55-cdf2-4708-a0cf-3c8d34ebbf7c kind: Event lastTimestamp: "2026-04-23T18:08:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:08:59Z" name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m.18a90ebe76b1b300 namespace: kserve-ci-e2e-test resourceVersion: "17256" uid: e63baff4-431e-42f9-b99a-13aa65514981 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:08:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: 405f9c55-cdf2-4708-a0cf-3c8d34ebbf7c kind: Event lastTimestamp: "2026-04-23T18:08:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:08:59Z" name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m.18a90ebe77260eac namespace: kserve-ci-e2e-test resourceVersion: "17258" uid: 7d7f67b8-82f6-4554-8b87-d75c2f9615f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:08:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: 405f9c55-cdf2-4708-a0cf-3c8d34ebbf7c kind: Event lastTimestamp: "2026-04-23T18:08:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:08:59Z" name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m.18a90ebe77318f5b namespace: kserve-ci-e2e-test resourceVersion: "17260" uid: 36f44b49-6da1-4a23-b224-6458028ca83f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:08:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: 405f9c55-cdf2-4708-a0cf-3c8d34ebbf7c kind: Event lastTimestamp: "2026-04-23T18:08:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:08:59Z" name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m.18a90ebe79d9170d namespace: kserve-ci-e2e-test resourceVersion: "17262" uid: dcd5b93c-7549-4101-9109-d112b19921ac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:08:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: 405f9c55-cdf2-4708-a0cf-3c8d34ebbf7c kind: Event lastTimestamp: "2026-04-23T18:08:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:08:59Z" name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m.18a90ebe7a4821ec namespace: kserve-ci-e2e-test resourceVersion: "17263" uid: c33f641a-b5c8-4f74-9ee7-e60f5af10fd4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 15 eventTime: null firstTimestamp: "2026-04-23T18:09:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: 405f9c55-cdf2-4708-a0cf-3c8d34ebbf7c kind: Event lastTimestamp: "2026-04-23T18:11:04Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:04Z" name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m.18a90ebeeb77d5a5 namespace: kserve-ci-e2e-test resourceVersion: "18160" uid: 7bb17593-a0d6-4e32-b4a3-c77ff1ca0e61 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: 405f9c55-cdf2-4708-a0cf-3c8d34ebbf7c kind: Event lastTimestamp: "2026-04-23T18:11:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:24Z" name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m.18a90ee058b290d1 namespace: kserve-ci-e2e-test resourceVersion: "18340" uid: 09ad3693-0fb1-48a6-85ec-a301ec491380 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: 405f9c55-cdf2-4708-a0cf-3c8d34ebbf7c kind: Event lastTimestamp: "2026-04-23T18:11:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:24Z" name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m.18a90ee058b367a8 namespace: kserve-ci-e2e-test resourceVersion: "18341" uid: 63b497f5-66b3-4520-a08a-ddeb69803929 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:11:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: 405f9c55-cdf2-4708-a0cf-3c8d34ebbf7c kind: Event lastTimestamp: "2026-04-23T18:11:32Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:11:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:32Z" name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m.18a90ee0ea05af1c namespace: kserve-ci-e2e-test resourceVersion: "18447" uid: 708867b4-5975-4f2d-a210-736663c0e421 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:08:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5 namespace: kserve-ci-e2e-test resourceVersion: "17152" uid: e8899926-1ad6-4fdc-b464-6bf0acac5576 kind: Event lastTimestamp: "2026-04-23T18:08:54Z" message: 'Created pod: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5-x2k4m' metadata: creationTimestamp: "2026-04-23T18:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:08:54Z" name: isvc-sklearn-scale-raw-8636b-predictor-57d74786b5.18a90ebd5ecd15f5 namespace: kserve-ci-e2e-test resourceVersion: "17179" uid: 29062350-19e6-4461-aa98-e6ab099b759f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:08:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-8636b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17145" uid: d01d3940-8c00-46b7-a3f9-ed7e83ccbd2e kind: Event lastTimestamp: "2026-04-23T18:08:54Z" message: Scaled up replica set isvc-sklearn-scale-raw-8636b-predictor-57d74786b5 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:08:54Z" name: isvc-sklearn-scale-raw-8636b-predictor.18a90ebd5b9b3cbb namespace: kserve-ci-e2e-test resourceVersion: "17157" uid: 2b31047c-9bea-4a4a-9927-0d90c2980330 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:09:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-8636b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17151" uid: c38bb82f-202f-4cbf-9243-e90576c92a85 kind: Event lastTimestamp: "2026-04-23T18:09:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:09:39Z" name: isvc-sklearn-scale-raw-8636b-predictor.18a90ec0da447103 namespace: kserve-ci-e2e-test resourceVersion: "17572" uid: c7097439-c98c-4e8e-af78-7ecb07547f85 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:09:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-8636b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17151" uid: c38bb82f-202f-4cbf-9243-e90576c92a85 kind: Event lastTimestamp: "2026-04-23T18:09:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:09:39Z" name: isvc-sklearn-scale-raw-8636b-predictor.18a90ec0dab2320e namespace: kserve-ci-e2e-test resourceVersion: "17573" uid: 73a919e6-c7ea-4f50-a72f-61e6b235ee95 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-23T18:09:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-8636b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: c38bb82f-202f-4cbf-9243-e90576c92a85 kind: Event lastTimestamp: "2026-04-23T18:11:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:09:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:11:24Z" name: isvc-sklearn-scale-raw-8636b-predictor.18a90ec459c7f5f3 namespace: kserve-ci-e2e-test resourceVersion: "18311" uid: b9ed9cad-3a6f-43ff-9bc9-ac713f010b7a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-23T18:09:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-8636b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: c38bb82f-202f-4cbf-9243-e90576c92a85 kind: Event lastTimestamp: "2026-04-23T18:11:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:09:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:11:24Z" name: isvc-sklearn-scale-raw-8636b-predictor.18a90ec45a318650 namespace: kserve-ci-e2e-test resourceVersion: "18312" uid: 04f05895-52d4-428b-b64a-e61b31922d45 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-23T18:08:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-8636b namespace: kserve-ci-e2e-test resourceVersion: "17136" uid: e609e712-3247-4661-aa47-cb17e79203d0 kind: Event lastTimestamp: "2026-04-23T18:11:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:11:14Z" name: isvc-sklearn-scale-raw-8636b.18a90ebd5b7a1f52 namespace: kserve-ci-e2e-test resourceVersion: "18243" uid: 34c14e3c-1d65-4a0c-a323-8b817f36f383 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:08:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-8636b namespace: kserve-ci-e2e-test resourceVersion: "17136" uid: e609e712-3247-4661-aa47-cb17e79203d0 kind: Event lastTimestamp: "2026-04-23T18:08:54Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-8636b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-8636b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:08:54Z" name: isvc-sklearn-scale-raw-8636b.18a90ebd5c3e012c namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: c8f43f93-f2e9-4e40-9064-99773918744d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-8636b namespace: kserve-ci-e2e-test resourceVersion: "18241" uid: e609e712-3247-4661-aa47-cb17e79203d0 kind: Event lastTimestamp: "2026-04-23T18:11:14Z" message: InferenceService [isvc-sklearn-scale-raw-8636b] is Ready metadata: creationTimestamp: "2026-04-23T18:11:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:11:14Z" name: isvc-sklearn-scale-raw-8636b.18a90eddf801a7d5 namespace: kserve-ci-e2e-test resourceVersion: "18242" uid: 36edbc1c-d70e-4853-b70b-0df5e58121a5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:03:58.705399Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf namespace: kserve-ci-e2e-test resourceVersion: "14394" uid: fe86fd31-2c47-4aac-869b-15a01eea4604 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf to ip-10-0-132-102.ec2.internal metadata: creationTimestamp: "2026-04-23T18:03:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:03:58Z" name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf.18a90e788178bf5d namespace: kserve-ci-e2e-test resourceVersion: "14402" uid: 9d2afed6-9d52-4444-82cf-435221eb3f16 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-774bdb58b-lw5wg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: fe86fd31-2c47-4aac-869b-15a01eea4604 kind: Event lastTimestamp: "2026-04-23T18:03:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-73769-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:03:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:58Z" name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf.18a90e788fa568b8 namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: 57a7f38e-6539-43f3-a980-2183d3d9524a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf namespace: kserve-ci-e2e-test resourceVersion: "14408" uid: fe86fd31-2c47-4aac-869b-15a01eea4604 kind: Event lastTimestamp: "2026-04-23T18:03:59Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:03:59Z" name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf.18a90e78bdd3bfdd namespace: kserve-ci-e2e-test resourceVersion: "14431" uid: 3035e03a-694b-4791-a1b3-9c05ccfd3dd9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: fe86fd31-2c47-4aac-869b-15a01eea4604 kind: Event lastTimestamp: "2026-04-23T18:03:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:48bcc0e7da25ba7560a82df7216c0ad464b268d851c665b85b9195165f27b89d" already present on machine metadata: creationTimestamp: "2026-04-23T18:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:59Z" name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf.18a90e78bf16ad5b namespace: kserve-ci-e2e-test resourceVersion: "14433" uid: ab7201e6-9f23-43c3-993d-e6cce9df180e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: fe86fd31-2c47-4aac-869b-15a01eea4604 kind: Event lastTimestamp: "2026-04-23T18:03:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T18:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:59Z" name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf.18a90e78c211b3b6 namespace: kserve-ci-e2e-test resourceVersion: "14434" uid: 7ae6416f-6195-4d8d-8590-191a15905b80 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: fe86fd31-2c47-4aac-869b-15a01eea4604 kind: Event lastTimestamp: "2026-04-23T18:03:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T18:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:03:59Z" name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf.18a90e78c27ccd26 namespace: kserve-ci-e2e-test resourceVersion: "14435" uid: 33588452-bc95-4aa3-95b1-2427aa3c8a38 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: fe86fd31-2c47-4aac-869b-15a01eea4604 kind: Event lastTimestamp: "2026-04-23T18:04:04Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-23T18:04:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:04Z" name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf.18a90e79cc6e2615 namespace: kserve-ci-e2e-test resourceVersion: "14485" uid: 5b22a841-fd27-449e-845e-6c11f54bb3ca reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: fe86fd31-2c47-4aac-869b-15a01eea4604 kind: Event lastTimestamp: "2026-04-23T18:04:20Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 16.272s (16.272s including waiting). Image size: 1306414326 bytes.' metadata: creationTimestamp: "2026-04-23T18:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:20Z" name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf.18a90e7d965d948c namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: e7a80f9a-c4d4-4bab-92d8-4188de401ffa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: fe86fd31-2c47-4aac-869b-15a01eea4604 kind: Event lastTimestamp: "2026-04-23T18:04:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:20Z" name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf.18a90e7d99941c2d namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 93c2c183-3e13-41a4-b0a5-090fad367160 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: fe86fd31-2c47-4aac-869b-15a01eea4604 kind: Event lastTimestamp: "2026-04-23T18:04:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:20Z" name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf.18a90e7d9a06f57e namespace: kserve-ci-e2e-test resourceVersion: "14628" uid: b53986b6-4b8d-447c-b068-3b5c472b0037 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: fe86fd31-2c47-4aac-869b-15a01eea4604 kind: Event lastTimestamp: "2026-04-23T18:04:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:20Z" name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf.18a90e7d9a12daf3 namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: b33e3183-5bff-4be2-ba7b-40a1d4c983ed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: fe86fd31-2c47-4aac-869b-15a01eea4604 kind: Event lastTimestamp: "2026-04-23T18:04:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:20Z" name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf.18a90e7d9d38aaeb namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: 83b971f4-1f1c-4491-b4fb-583be672d029 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: fe86fd31-2c47-4aac-869b-15a01eea4604 kind: Event lastTimestamp: "2026-04-23T18:04:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:04:20Z" name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf.18a90e7d9da80375 namespace: kserve-ci-e2e-test resourceVersion: "14631" uid: 9fd05987-2ea7-4a71-a99f-092d5f80567e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-23T18:04:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: fe86fd31-2c47-4aac-869b-15a01eea4604 kind: Event lastTimestamp: "2026-04-23T18:05:18Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:04:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:18Z" name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf.18a90e7e00e09746 namespace: kserve-ci-e2e-test resourceVersion: "15066" uid: c8999b8f-ae30-469d-a550-d1aff2c1e2a5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: fe86fd31-2c47-4aac-869b-15a01eea4604 kind: Event lastTimestamp: "2026-04-23T18:05:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:39Z" name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf.18a90e8fdf13b624 namespace: kserve-ci-e2e-test resourceVersion: "15370" uid: 2880b3bb-8e1e-4dc3-a567-5cef73fe7576 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: fe86fd31-2c47-4aac-869b-15a01eea4604 kind: Event lastTimestamp: "2026-04-23T18:05:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:39Z" name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf.18a90e8fdf141375 namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: 993ebc5b-8005-42ec-ac8a-fc2cb02e5209 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9 namespace: kserve-ci-e2e-test resourceVersion: "14384" uid: 4c18408a-dcc1-47b9-88de-d6c0455df856 kind: Event lastTimestamp: "2026-04-23T18:03:58Z" message: 'Created pod: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9-rnljf' metadata: creationTimestamp: "2026-04-23T18:03:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:58Z" name: isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9.18a90e78810576f9 namespace: kserve-ci-e2e-test resourceVersion: "14397" uid: d927922e-10b1-4a3e-96b6-b37b5634800b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:03:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-73769-predictor namespace: kserve-ci-e2e-test resourceVersion: "14382" uid: 74d67b58-583b-4517-bb12-5b4e4c5307e9 kind: Event lastTimestamp: "2026-04-23T18:03:58Z" message: Scaled up replica set isvc-xgboost-graph-raw-73769-predictor-7ff67dc9b9 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:03:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:03:58Z" name: isvc-xgboost-graph-raw-73769-predictor.18a90e787e5a192a namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: 02c221e3-3d70-4e13-b550-70ab3e97e9e0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:04:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-73769-predictor namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: 58fe8913-97ee-4ac4-a1e0-c5ae6c9d6edf kind: Event lastTimestamp: "2026-04-23T18:04:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:04:43Z" name: isvc-xgboost-graph-raw-73769-predictor.18a90e7bfcd5c505 namespace: kserve-ci-e2e-test resourceVersion: "14807" uid: d38b10bd-ded8-4326-8e11-326000e11c40 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:04:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-73769-predictor namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: 58fe8913-97ee-4ac4-a1e0-c5ae6c9d6edf kind: Event lastTimestamp: "2026-04-23T18:04:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:04:43Z" name: isvc-xgboost-graph-raw-73769-predictor.18a90e7bfd4df132 namespace: kserve-ci-e2e-test resourceVersion: "14808" uid: 75c9b1e7-47f5-43cb-a2a8-9ebce8bc08ee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:04:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-73769-predictor namespace: kserve-ci-e2e-test resourceVersion: "14569" uid: 58fe8913-97ee-4ac4-a1e0-c5ae6c9d6edf kind: Event lastTimestamp: "2026-04-23T18:05:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:04:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:05:28Z" name: isvc-xgboost-graph-raw-73769-predictor.18a90e8679877879 namespace: kserve-ci-e2e-test resourceVersion: "15177" uid: ea62ca44-eb47-4b76-9785-f46c72202b56 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:04:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-73769-predictor namespace: kserve-ci-e2e-test resourceVersion: "14569" uid: 58fe8913-97ee-4ac4-a1e0-c5ae6c9d6edf kind: Event lastTimestamp: "2026-04-23T18:05:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:04:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:05:28Z" name: isvc-xgboost-graph-raw-73769-predictor.18a90e8679f3d540 namespace: kserve-ci-e2e-test resourceVersion: "15178" uid: 8437bd25-fac1-434c-9b00-229b21055ec9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T18:03:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-73769 namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: 38bb67fc-bf1a-4f49-bc42-f9b2bc4d8124 kind: Event lastTimestamp: "2026-04-23T18:05:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:03:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:05:28Z" name: isvc-xgboost-graph-raw-73769.18a90e787e2f351e namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: fcc15758-1db7-4c6a-817b-10f5c4a67121 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-73769 namespace: kserve-ci-e2e-test resourceVersion: "15139" uid: 38bb67fc-bf1a-4f49-bc42-f9b2bc4d8124 kind: Event lastTimestamp: "2026-04-23T18:05:28Z" message: InferenceService [isvc-xgboost-graph-raw-73769] is Ready metadata: creationTimestamp: "2026-04-23T18:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:05:28Z" name: isvc-xgboost-graph-raw-73769.18a90e8d642e8afd namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: 9fffb746-ded9-46df-a0a0-117158d421fc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:05:39.069106Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x namespace: kserve-ci-e2e-test resourceVersion: "15366" uid: 12caa729-dcf3-4776-aa94-964438c08d3e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x to ip-10-0-132-102.ec2.internal metadata: creationTimestamp: "2026-04-23T18:05:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:05:39Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x.18a90e8fdf9d5120 namespace: kserve-ci-e2e-test resourceVersion: "15373" uid: 367fd011-e26d-4a45-b595-26b528bbc5d8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-774bdb58b-lw5wg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: 12caa729-dcf3-4776-aa94-964438c08d3e kind: Event lastTimestamp: "2026-04-23T18:05:39Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:05:39Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x.18a90e8ff8537196 namespace: kserve-ci-e2e-test resourceVersion: "15385" uid: d50ac90e-b778-4540-ad00-a5ffa5feed09 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: 12caa729-dcf3-4776-aa94-964438c08d3e kind: Event lastTimestamp: "2026-04-23T18:05:39Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:48bcc0e7da25ba7560a82df7216c0ad464b268d851c665b85b9195165f27b89d" already present on machine metadata: creationTimestamp: "2026-04-23T18:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:39Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x.18a90e8ff9a6212e namespace: kserve-ci-e2e-test resourceVersion: "15388" uid: 31cc2246-e780-4cac-8729-1ef5ddd21f3e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: 12caa729-dcf3-4776-aa94-964438c08d3e kind: Event lastTimestamp: "2026-04-23T18:05:39Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T18:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:39Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x.18a90e8ffc2d753d namespace: kserve-ci-e2e-test resourceVersion: "15390" uid: 470293f9-eba3-491a-a099-d257238a9296 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: 12caa729-dcf3-4776-aa94-964438c08d3e kind: Event lastTimestamp: "2026-04-23T18:05:39Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T18:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:39Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x.18a90e8ffc89c179 namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 361770c1-d9d5-448a-b6fc-5289d50865d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: 12caa729-dcf3-4776-aa94-964438c08d3e kind: Event lastTimestamp: "2026-04-23T18:05:43Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-23T18:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:43Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x.18a90e90ea8f44b1 namespace: kserve-ci-e2e-test resourceVersion: "15449" uid: 1b7c55db-c79e-4a3d-bcae-ed762020a2a6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: 12caa729-dcf3-4776-aa94-964438c08d3e kind: Event lastTimestamp: "2026-04-23T18:05:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:43Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x.18a90e90ed8a91ea namespace: kserve-ci-e2e-test resourceVersion: "15457" uid: ac69df23-2e6d-48dd-bcaa-d2ea18cf6cab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: 12caa729-dcf3-4776-aa94-964438c08d3e kind: Event lastTimestamp: "2026-04-23T18:05:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:43Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x.18a90e90ee249e1f namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: 03b0f107-1ab0-4832-8a01-e6e7545b1222 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: 12caa729-dcf3-4776-aa94-964438c08d3e kind: Event lastTimestamp: "2026-04-23T18:05:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:43Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x.18a90e90ee316744 namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: 3dd6117e-f229-48b3-a21a-440332f95673 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: 12caa729-dcf3-4776-aa94-964438c08d3e kind: Event lastTimestamp: "2026-04-23T18:05:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:43Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x.18a90e90f1678766 namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: b9605a24-8e34-4ba2-9ab8-b52b3cbb2c43 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: 12caa729-dcf3-4776-aa94-964438c08d3e kind: Event lastTimestamp: "2026-04-23T18:05:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:43Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x.18a90e90f1d951e4 namespace: kserve-ci-e2e-test resourceVersion: "15469" uid: a4e86c2e-9df5-4164-acb0-9324d6da014b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T18:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: 12caa729-dcf3-4776-aa94-964438c08d3e kind: Event lastTimestamp: "2026-04-23T18:07:20Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:20Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x.18a90e9126d0bd85 namespace: kserve-ci-e2e-test resourceVersion: "16347" uid: 5d9b6756-ffb3-41e6-929b-4110718f4b39 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: 12caa729-dcf3-4776-aa94-964438c08d3e kind: Event lastTimestamp: "2026-04-23T18:07:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:19Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x.18a90ea732f2cc0b namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 2215ff68-392e-42c7-8e7c-ba0b8cced585 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: 12caa729-dcf3-4776-aa94-964438c08d3e kind: Event lastTimestamp: "2026-04-23T18:07:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:19Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x.18a90ea732f36182 namespace: kserve-ci-e2e-test resourceVersion: "16331" uid: 987cd9ce-519d-4cd8-b297-17e7a0b971a2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: 12caa729-dcf3-4776-aa94-964438c08d3e kind: Event lastTimestamp: "2026-04-23T18:07:20Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:20Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x.18a90ea781154b3d namespace: kserve-ci-e2e-test resourceVersion: "16346" uid: 72022910-ff63-44f4-a461-5f4f5905a576 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677 namespace: kserve-ci-e2e-test resourceVersion: "15338" uid: 4d8eb393-62f8-471b-99bf-53eb5cec4b5e kind: Event lastTimestamp: "2026-04-23T18:05:39Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677-g4k5x' metadata: creationTimestamp: "2026-04-23T18:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:05:39Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677.18a90e8fdf01d9b0 namespace: kserve-ci-e2e-test resourceVersion: "15368" uid: 69dfbde4-8c58-42ae-969a-27cf2d54169c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-050b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: 369d1abc-f743-4c26-9744-56f6da696e1f kind: Event lastTimestamp: "2026-04-23T18:05:38Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-050b4-predictor-5d6f74677 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:05:38Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor.18a90e8fd70b6341 namespace: kserve-ci-e2e-test resourceVersion: "15342" uid: c7b502c6-08ce-4662-bf6a-6063220226d5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:05:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-050b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: a8ccfced-445a-4c82-b6f8-1f5846129206 kind: Event lastTimestamp: "2026-04-23T18:06:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:05:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:06:23Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor.18a90e9355409d7c namespace: kserve-ci-e2e-test resourceVersion: "15785" uid: 1fbafffc-006a-4fad-b825-17d9130d6098 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:05:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-050b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: a8ccfced-445a-4c82-b6f8-1f5846129206 kind: Event lastTimestamp: "2026-04-23T18:06:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:05:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:06:23Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor.18a90e9355a7cb5a namespace: kserve-ci-e2e-test resourceVersion: "15786" uid: 2f874e08-89a7-44af-88bc-65ddf3ab42e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:06:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-050b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: a8ccfced-445a-4c82-b6f8-1f5846129206 kind: Event lastTimestamp: "2026-04-23T18:07:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:06:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:07:09Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor.18a90e96d469a2df namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: 0300f784-712c-4193-8979-89e9b04017c9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:06:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-050b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: a8ccfced-445a-4c82-b6f8-1f5846129206 kind: Event lastTimestamp: "2026-04-23T18:07:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:06:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:07:09Z" name: isvc-xgboost-graph-raw-hpa-050b4-predictor.18a90e96d4de76d0 namespace: kserve-ci-e2e-test resourceVersion: "16153" uid: 7e089d0e-55f8-480e-bc00-88e0a615cf20 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T18:05:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-050b4 namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 1d6f8040-5c02-4d0f-adc4-2b80d1f2a197 kind: Event lastTimestamp: "2026-04-23T18:06:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:06:50Z" name: isvc-xgboost-graph-raw-hpa-050b4.18a90e8fd69cd990 namespace: kserve-ci-e2e-test resourceVersion: "15988" uid: 7668667b-7971-4777-a238-3ba8c6b0c93b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-050b4 namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 1d6f8040-5c02-4d0f-adc4-2b80d1f2a197 kind: Event lastTimestamp: "2026-04-23T18:05:38Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-050b4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-050b4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:05:38Z" name: isvc-xgboost-graph-raw-hpa-050b4.18a90e8fd748e6a1 namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: 097ea6f0-21d8-4677-8d58-060298596523 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:06:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-050b4 namespace: kserve-ci-e2e-test resourceVersion: "15986" uid: 1d6f8040-5c02-4d0f-adc4-2b80d1f2a197 kind: Event lastTimestamp: "2026-04-23T18:06:50Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-050b4] is Ready metadata: creationTimestamp: "2026-04-23T18:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:06:50Z" name: isvc-xgboost-graph-raw-hpa-050b4.18a90ea089fa3aa1 namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 3d6abc80-e0ab-4ead-a319-0caee6752243 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:07:19.202640Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2 namespace: kserve-ci-e2e-test resourceVersion: "16301" uid: e6861b4e-d7a6-4e70-9d96-fd8771c8c90a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2 to ip-10-0-132-102.ec2.internal metadata: creationTimestamp: "2026-04-23T18:07:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:07:19Z" name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2.18a90ea73009c2b3 namespace: kserve-ci-e2e-test resourceVersion: "16308" uid: d0e96cb4-0945-4d08-90b7-655c5aec70d8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-774bdb58b-lw5wg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:19Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2 namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: e6861b4e-d7a6-4e70-9d96-fd8771c8c90a kind: Event lastTimestamp: "2026-04-23T18:07:19Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:07:19Z" name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2.18a90ea748b615b5 namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: 5c9b777c-2c68-4801-acd7-b5da8656163c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2 namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: e6861b4e-d7a6-4e70-9d96-fd8771c8c90a kind: Event lastTimestamp: "2026-04-23T18:07:19Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-23T18:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:19Z" name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2.18a90ea749ee0e31 namespace: kserve-ci-e2e-test resourceVersion: "16340" uid: de8ca9a4-0f03-466a-9c27-b243935162ee reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2 namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: e6861b4e-d7a6-4e70-9d96-fd8771c8c90a kind: Event lastTimestamp: "2026-04-23T18:07:20Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 937ms (937ms including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-23T18:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:20Z" name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2.18a90ea781ce8e09 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: a0aa2e34-2d74-44e0-aa8e-90543938cc01 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2 namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: e6861b4e-d7a6-4e70-9d96-fd8771c8c90a kind: Event lastTimestamp: "2026-04-23T18:07:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:20Z" name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2.18a90ea784c924b8 namespace: kserve-ci-e2e-test resourceVersion: "16349" uid: 847c753e-4104-40cd-8c90-685228e9e7f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2 namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: e6861b4e-d7a6-4e70-9d96-fd8771c8c90a kind: Event lastTimestamp: "2026-04-23T18:07:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:20Z" name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2.18a90ea7853b786a namespace: kserve-ci-e2e-test resourceVersion: "16351" uid: 68dcb2a3-3122-495a-8c6c-8fba4a20f0e4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2 namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: e6861b4e-d7a6-4e70-9d96-fd8771c8c90a kind: Event lastTimestamp: "2026-04-23T18:07:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:20Z" name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2.18a90ea78547a1c7 namespace: kserve-ci-e2e-test resourceVersion: "16352" uid: 537b1afd-08b6-4f3f-98ef-088a7c61b960 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2 namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: e6861b4e-d7a6-4e70-9d96-fd8771c8c90a kind: Event lastTimestamp: "2026-04-23T18:07:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:20Z" name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2.18a90ea78817526b namespace: kserve-ci-e2e-test resourceVersion: "16354" uid: 2ff42263-2426-4f20-b836-ac888605b780 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2 namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: e6861b4e-d7a6-4e70-9d96-fd8771c8c90a kind: Event lastTimestamp: "2026-04-23T18:07:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:20Z" name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2.18a90ea78882eb1a namespace: kserve-ci-e2e-test resourceVersion: "16355" uid: 2b2b7964-5c67-42f1-8d33-cdb18bbbb3af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:08:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2 namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: e6861b4e-d7a6-4e70-9d96-fd8771c8c90a kind: Event lastTimestamp: "2026-04-23T18:08:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:08:54Z" name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2.18a90ebd63a014be namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 27e24d55-4ad8-4d17-bf4e-bdabffe6e334 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:08:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2 namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: e6861b4e-d7a6-4e70-9d96-fd8771c8c90a kind: Event lastTimestamp: "2026-04-23T18:08:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:08:54Z" name: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2.18a90ebd63a097a2 namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: f1387e14-9eb6-4fba-812f-5e31c5ab4f57 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-f1c6b-predictor-5bf88487b4 namespace: kserve-ci-e2e-test resourceVersion: "16285" uid: d3aeeb35-eb11-4a61-a077-957fe2e25641 kind: Event lastTimestamp: "2026-04-23T18:07:19Z" message: 'Created pod: message-dumper-raw-f1c6b-predictor-5bf88487b4-vdqr2' metadata: creationTimestamp: "2026-04-23T18:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:07:19Z" name: message-dumper-raw-f1c6b-predictor-5bf88487b4.18a90ea72f78d00d namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 5e29d20b-4db3-45e3-8d30-37a6077bd3a8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-f1c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 99ada42f-378b-40f6-8624-6afd10c712aa kind: Event lastTimestamp: "2026-04-23T18:07:19Z" message: Scaled up replica set message-dumper-raw-f1c6b-predictor-5bf88487b4 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:07:19Z" name: message-dumper-raw-f1c6b-predictor.18a90ea72c428e3a namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: c02f1afd-94cb-42cf-8280-56a505f16dc9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:07:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-f1c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: 05685d84-7fa6-44da-be12-1e3bf2734cb2 kind: Event lastTimestamp: "2026-04-23T18:08:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:07:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:08:04Z" name: message-dumper-raw-f1c6b-predictor.18a90eaaab567ad5 namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: f0c76002-b913-44e1-8970-dbb1a5dcb236 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:07:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-f1c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: 05685d84-7fa6-44da-be12-1e3bf2734cb2 kind: Event lastTimestamp: "2026-04-23T18:08:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:07:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:08:04Z" name: message-dumper-raw-f1c6b-predictor.18a90eaaaba3d571 namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: 7a8f64ca-2525-4710-9648-ee5ec7ef0bb1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T18:07:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-f1c6b namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: 74835157-e475-4ebb-b078-949e69d160b2 kind: Event lastTimestamp: "2026-04-23T18:07:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:07:28Z" name: message-dumper-raw-f1c6b.18a90ea72c473a91 namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: 5f611164-0dfb-4495-bac5-16b203676fd5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-f1c6b namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: 74835157-e475-4ebb-b078-949e69d160b2 kind: Event lastTimestamp: "2026-04-23T18:07:19Z" message: 'Failed to update status for InferenceService "message-dumper-raw-f1c6b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-f1c6b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:07:19Z" name: message-dumper-raw-f1c6b.18a90ea72ce71237 namespace: kserve-ci-e2e-test resourceVersion: "16294" uid: ca666688-5f53-4a53-a2d5-feee52b44dbb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-f1c6b namespace: kserve-ci-e2e-test resourceVersion: "16432" uid: 74835157-e475-4ebb-b078-949e69d160b2 kind: Event lastTimestamp: "2026-04-23T18:07:28Z" message: InferenceService [message-dumper-raw-f1c6b] is Ready metadata: creationTimestamp: "2026-04-23T18:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:07:28Z" name: message-dumper-raw-f1c6b.18a90ea973fed146 namespace: kserve-ci-e2e-test resourceVersion: "16433" uid: c9a21f92-b33d-4ff5-9c4a-555091d9551b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:05:28.637107Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-73769-75489dcf9b-h4fqh namespace: kserve-ci-e2e-test resourceVersion: "15157" uid: 2870f85b-d914-487a-bfba-88359ee13c8d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-73769-75489dcf9b-h4fqh to ip-10-0-130-202.ec2.internal metadata: creationTimestamp: "2026-04-23T18:05:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:05:28Z" name: model-chainer-raw-73769-75489dcf9b-h4fqh.18a90e8d71d19ef9 namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 85f19bb1-dc70-41a7-835b-58bd44c1a8b7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-774bdb58b-lw5wg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:28Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-73769-75489dcf9b-h4fqh namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 2870f85b-d914-487a-bfba-88359ee13c8d kind: Event lastTimestamp: "2026-04-23T18:05:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-73769-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:28Z" name: model-chainer-raw-73769-75489dcf9b-h4fqh.18a90e8d7d13109f namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: 901511a0-733a-4605-8aea-1b29e1f12128 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-202.ec2.internal source: component: kubelet host: ip-10-0-130-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:29Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-73769-75489dcf9b-h4fqh namespace: kserve-ci-e2e-test resourceVersion: "15168" uid: 2870f85b-d914-487a-bfba-88359ee13c8d kind: Event lastTimestamp: "2026-04-23T18:05:29Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:05:29Z" name: model-chainer-raw-73769-75489dcf9b-h4fqh.18a90e8daeba52f0 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 3d9a7e0f-4c3c-4d24-be0f-b6498f03de45 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-73769} kind: Pod name: model-chainer-raw-73769-75489dcf9b-h4fqh namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 2870f85b-d914-487a-bfba-88359ee13c8d kind: Event lastTimestamp: "2026-04-23T18:05:29Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:013b59be0bfa982c0e31bad4b53ad46f4b1067e5dae5be631dfd76f63f738c54" metadata: creationTimestamp: "2026-04-23T18:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:29Z" name: model-chainer-raw-73769-75489dcf9b-h4fqh.18a90e8db00671ec namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: df56d954-aa7a-4b85-9828-4afe8cba14f8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-202.ec2.internal source: component: kubelet host: ip-10-0-130-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-73769} kind: Pod name: model-chainer-raw-73769-75489dcf9b-h4fqh namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 2870f85b-d914-487a-bfba-88359ee13c8d kind: Event lastTimestamp: "2026-04-23T18:05:32Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:013b59be0bfa982c0e31bad4b53ad46f4b1067e5dae5be631dfd76f63f738c54" in 2.777s (2.777s including waiting). Image size: 216346204 bytes.' metadata: creationTimestamp: "2026-04-23T18:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:32Z" name: model-chainer-raw-73769-75489dcf9b-h4fqh.18a90e8e5596509c namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: b0e147e5-7070-441b-8d61-661d25017244 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-202.ec2.internal source: component: kubelet host: ip-10-0-130-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-73769} kind: Pod name: model-chainer-raw-73769-75489dcf9b-h4fqh namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 2870f85b-d914-487a-bfba-88359ee13c8d kind: Event lastTimestamp: "2026-04-23T18:05:32Z" message: 'Created container: model-chainer-raw-73769' metadata: creationTimestamp: "2026-04-23T18:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:32Z" name: model-chainer-raw-73769-75489dcf9b-h4fqh.18a90e8e589d43ea namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: 29749e7d-d569-449f-a412-1fc7da6571f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-202.ec2.internal source: component: kubelet host: ip-10-0-130-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-73769} kind: Pod name: model-chainer-raw-73769-75489dcf9b-h4fqh namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 2870f85b-d914-487a-bfba-88359ee13c8d kind: Event lastTimestamp: "2026-04-23T18:05:32Z" message: Started container model-chainer-raw-73769 metadata: creationTimestamp: "2026-04-23T18:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:32Z" name: model-chainer-raw-73769-75489dcf9b-h4fqh.18a90e8e59164385 namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: c258fc50-c300-4ce9-9fc5-d8ccb06a13cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-202.ec2.internal source: component: kubelet host: ip-10-0-130-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-73769} kind: Pod name: model-chainer-raw-73769-75489dcf9b-h4fqh namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 2870f85b-d914-487a-bfba-88359ee13c8d kind: Event lastTimestamp: "2026-04-23T18:05:38Z" message: Stopping container model-chainer-raw-73769 metadata: creationTimestamp: "2026-04-23T18:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:05:38Z" name: model-chainer-raw-73769-75489dcf9b-h4fqh.18a90e8fc924f7fe namespace: kserve-ci-e2e-test resourceVersion: "15271" uid: caf5f668-430e-48b8-a2ba-38278f7ec530 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-202.ec2.internal source: component: kubelet host: ip-10-0-130-202.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T18:05:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-73769} kind: Pod name: model-chainer-raw-73769-75489dcf9b-h4fqh namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 2870f85b-d914-487a-bfba-88359ee13c8d kind: Event lastTimestamp: "2026-04-23T18:06:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:06:08Z" name: model-chainer-raw-73769-75489dcf9b-h4fqh.18a90e8fc994208c namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: f2eb9368-709f-4745-a306-9ff467b5848e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-202.ec2.internal source: component: kubelet host: ip-10-0-130-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-73769-75489dcf9b namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: e3af4856-ffb8-4ddb-97e7-48d9c15682a4 kind: Event lastTimestamp: "2026-04-23T18:05:28Z" message: 'Created pod: model-chainer-raw-73769-75489dcf9b-h4fqh' metadata: creationTimestamp: "2026-04-23T18:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:05:28Z" name: model-chainer-raw-73769-75489dcf9b.18a90e8d7140f706 namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 1ce6e0ee-7ffc-47ad-9beb-6cb2b6d0b81a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-73769 namespace: kserve-ci-e2e-test resourceVersion: "15152" uid: 4acc7ba8-102f-46bc-8af6-cc807f64e825 kind: Event lastTimestamp: "2026-04-23T18:05:28Z" message: Scaled up replica set model-chainer-raw-73769-75489dcf9b from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:05:28Z" name: model-chainer-raw-73769.18a90e8d706c2456 namespace: kserve-ci-e2e-test resourceVersion: "15155" uid: 0189bf8e-b8a7-4df5-8f6c-a1936af3835d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:05:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-73769 namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: 6a28d6e8-8327-4dd8-b027-9eb1fc0937a2 kind: Event lastTimestamp: "2026-04-23T18:05:28Z" message: InferenceGraph [model-chainer-raw-73769] is Ready metadata: creationTimestamp: "2026-04-23T18:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:05:28Z" name: model-chainer-raw-73769.18a90e8d74530f9d namespace: kserve-ci-e2e-test resourceVersion: "15173" uid: b403a3df-f9c9-41a7-96cc-1be58d335474 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:07:08.940285Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-050b4-56df766c48-ckj2f namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: 6dcc1f0c-6923-4d2d-a878-33a4ba3f6d94 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-050b4-56df766c48-ckj2f to ip-10-0-130-202.ec2.internal metadata: creationTimestamp: "2026-04-23T18:07:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:07:08Z" name: model-chainer-raw-hpa-050b4-56df766c48-ckj2f.18a90ea4cc5ab250 namespace: kserve-ci-e2e-test resourceVersion: "16143" uid: 0e838896-a5c3-4e05-b9e5-6f7fc5c62a13 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-774bdb58b-lw5wg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:09Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-050b4-56df766c48-ckj2f namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: 6dcc1f0c-6923-4d2d-a878-33a4ba3f6d94 kind: Event lastTimestamp: "2026-04-23T18:07:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-050b4-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:09Z" name: model-chainer-raw-hpa-050b4-56df766c48-ckj2f.18a90ea4d5405f3e namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 453a1c94-1b06-430f-a2ba-f6f0cdacb639 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-202.ec2.internal source: component: kubelet host: ip-10-0-130-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:09Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-050b4-56df766c48-ckj2f namespace: kserve-ci-e2e-test resourceVersion: "16147" uid: 6dcc1f0c-6923-4d2d-a878-33a4ba3f6d94 kind: Event lastTimestamp: "2026-04-23T18:07:09Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:07:09Z" name: model-chainer-raw-hpa-050b4-56df766c48-ckj2f.18a90ea50a8c8276 namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: f49dda98-47ba-4aeb-b0c3-1e35d1938be7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-050b4} kind: Pod name: model-chainer-raw-hpa-050b4-56df766c48-ckj2f namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: 6dcc1f0c-6923-4d2d-a878-33a4ba3f6d94 kind: Event lastTimestamp: "2026-04-23T18:07:10Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:013b59be0bfa982c0e31bad4b53ad46f4b1067e5dae5be631dfd76f63f738c54" already present on machine metadata: creationTimestamp: "2026-04-23T18:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:10Z" name: model-chainer-raw-hpa-050b4-56df766c48-ckj2f.18a90ea50bdd1cf3 namespace: kserve-ci-e2e-test resourceVersion: "16168" uid: 5aa2a35e-3ef5-4ecc-82d9-07c04cdf9f1c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-202.ec2.internal source: component: kubelet host: ip-10-0-130-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-050b4} kind: Pod name: model-chainer-raw-hpa-050b4-56df766c48-ckj2f namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: 6dcc1f0c-6923-4d2d-a878-33a4ba3f6d94 kind: Event lastTimestamp: "2026-04-23T18:07:10Z" message: 'Created container: model-chainer-raw-hpa-050b4' metadata: creationTimestamp: "2026-04-23T18:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:10Z" name: model-chainer-raw-hpa-050b4-56df766c48-ckj2f.18a90ea50fb20a14 namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: e9d6c826-179b-4367-8c6e-b44546322b1a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-202.ec2.internal source: component: kubelet host: ip-10-0-130-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-050b4} kind: Pod name: model-chainer-raw-hpa-050b4-56df766c48-ckj2f namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: 6dcc1f0c-6923-4d2d-a878-33a4ba3f6d94 kind: Event lastTimestamp: "2026-04-23T18:07:10Z" message: Started container model-chainer-raw-hpa-050b4 metadata: creationTimestamp: "2026-04-23T18:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:10Z" name: model-chainer-raw-hpa-050b4-56df766c48-ckj2f.18a90ea510276bc3 namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: b40e7eaf-135c-435e-b8ab-5aecd79ef563 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-202.ec2.internal source: component: kubelet host: ip-10-0-130-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-050b4} kind: Pod name: model-chainer-raw-hpa-050b4-56df766c48-ckj2f namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: 6dcc1f0c-6923-4d2d-a878-33a4ba3f6d94 kind: Event lastTimestamp: "2026-04-23T18:07:19Z" message: Stopping container model-chainer-raw-hpa-050b4 metadata: creationTimestamp: "2026-04-23T18:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:19Z" name: model-chainer-raw-hpa-050b4-56df766c48-ckj2f.18a90ea724ad8756 namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: 2f90496a-966f-4a58-9e2c-e35e9c0a9de0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-202.ec2.internal source: component: kubelet host: ip-10-0-130-202.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:07:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-050b4} kind: Pod name: model-chainer-raw-hpa-050b4-56df766c48-ckj2f namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: 6dcc1f0c-6923-4d2d-a878-33a4ba3f6d94 kind: Event lastTimestamp: "2026-04-23T18:07:46Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:07:46Z" name: model-chainer-raw-hpa-050b4-56df766c48-ckj2f.18a90ea7ac2a6485 namespace: kserve-ci-e2e-test resourceVersion: "16635" uid: 88963e62-8d4e-4236-9d91-4bcb5eff2de3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-202.ec2.internal source: component: kubelet host: ip-10-0-130-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-050b4-56df766c48 namespace: kserve-ci-e2e-test resourceVersion: "16133" uid: 7e2a32da-9681-4ddd-9104-a68d94cb8518 kind: Event lastTimestamp: "2026-04-23T18:07:08Z" message: 'Created pod: model-chainer-raw-hpa-050b4-56df766c48-ckj2f' metadata: creationTimestamp: "2026-04-23T18:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:07:08Z" name: model-chainer-raw-hpa-050b4-56df766c48.18a90ea4cbf3e85c namespace: kserve-ci-e2e-test resourceVersion: "16138" uid: 3824f2a4-b184-4c43-aeff-f8beba565b1a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-050b4 namespace: kserve-ci-e2e-test resourceVersion: "16131" uid: 9e25db3e-5988-4719-a7bb-2e823ea466fb kind: Event lastTimestamp: "2026-04-23T18:07:08Z" message: Scaled up replica set model-chainer-raw-hpa-050b4-56df766c48 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:07:08Z" name: model-chainer-raw-hpa-050b4.18a90ea4cb2fe266 namespace: kserve-ci-e2e-test resourceVersion: "16134" uid: 605bf106-733a-4d58-9945-d103c44139a3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:08Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-050b4 namespace: kserve-ci-e2e-test resourceVersion: "16149" uid: f9ed9d4e-5930-483c-93bb-e3e9f1fe15b1 kind: Event lastTimestamp: "2026-04-23T18:07:08Z" message: InferenceGraph [model-chainer-raw-hpa-050b4] is Ready metadata: creationTimestamp: "2026-04-23T18:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:07:08Z" name: model-chainer-raw-hpa-050b4.18a90ea4cefa055f namespace: kserve-ci-e2e-test resourceVersion: "16151" uid: 75651e8b-e8ac-46dc-a842-1b2051c768b8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-050b4 namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: f9ed9d4e-5930-483c-93bb-e3e9f1fe15b1 kind: Event lastTimestamp: "2026-04-23T18:07:09Z" message: 'Failed to update status for InferenceGraph "model-chainer-raw-hpa-050b4": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-hpa-050b4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:07:09Z" name: model-chainer-raw-hpa-050b4.18a90ea4d176b90c namespace: kserve-ci-e2e-test resourceVersion: "16155" uid: 90f63feb-3dd4-4d2f-a02c-69bdf3dd9548 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:07:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-050b4 namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: f9ed9d4e-5930-483c-93bb-e3e9f1fe15b1 kind: Event lastTimestamp: "2026-04-23T18:07:09Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-hpa-050b4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:07:09Z" name: model-chainer-raw-hpa-050b4.18a90ea4d1770cea namespace: kserve-ci-e2e-test resourceVersion: "16156" uid: c3b0f832-dfc8-44aa-b759-ab0c004d914c reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:13:11.242587Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5 namespace: kserve-ci-e2e-test resourceVersion: "19374" uid: 2d4b9978-964d-419e-b6c2-4693a1625360 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5 to ip-10-0-132-102.ec2.internal metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:13:11Z" name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5.18a90ef9274127d8 namespace: kserve-ci-e2e-test resourceVersion: "19380" uid: 794208ed-323f-448d-8c83-b0a029c146a9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-774bdb58b-lw5wg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5 namespace: kserve-ci-e2e-test resourceVersion: "19385" uid: 2d4b9978-964d-419e-b6c2-4693a1625360 kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:13:11Z" name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5.18a90ef9403006f8 namespace: kserve-ci-e2e-test resourceVersion: "19393" uid: 7c672763-0fc7-4a2d-b1f3-3e110392e654 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5 namespace: kserve-ci-e2e-test resourceVersion: "19377" uid: 2d4b9978-964d-419e-b6c2-4693a1625360 kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:48bcc0e7da25ba7560a82df7216c0ad464b268d851c665b85b9195165f27b89d" already present on machine metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:11Z" name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5.18a90ef9417265b6 namespace: kserve-ci-e2e-test resourceVersion: "19395" uid: 268d46f9-eb28-4e6a-9149-b56e26731856 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5 namespace: kserve-ci-e2e-test resourceVersion: "19377" uid: 2d4b9978-964d-419e-b6c2-4693a1625360 kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:11Z" name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5.18a90ef9440b4777 namespace: kserve-ci-e2e-test resourceVersion: "19397" uid: ba0b4af0-c711-49d3-ad82-2fcc09afcb48 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5 namespace: kserve-ci-e2e-test resourceVersion: "19377" uid: 2d4b9978-964d-419e-b6c2-4693a1625360 kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:11Z" name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5.18a90ef9447cf328 namespace: kserve-ci-e2e-test resourceVersion: "19398" uid: c28c089a-a45a-494c-a29f-af75f648a24f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5 namespace: kserve-ci-e2e-test resourceVersion: "19377" uid: 2d4b9978-964d-419e-b6c2-4693a1625360 kind: Event lastTimestamp: "2026-04-23T18:13:15Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:13:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:15Z" name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5.18a90efa3acc5dd1 namespace: kserve-ci-e2e-test resourceVersion: "19434" uid: 0748c5e4-9597-4da4-b8ab-d7342c4f8e9c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5 namespace: kserve-ci-e2e-test resourceVersion: "19377" uid: 2d4b9978-964d-419e-b6c2-4693a1625360 kind: Event lastTimestamp: "2026-04-23T18:13:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:13:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:15Z" name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5.18a90efa3da6a7c9 namespace: kserve-ci-e2e-test resourceVersion: "19436" uid: fd9eb323-41bc-4d70-9c8f-56a993c7f01a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5 namespace: kserve-ci-e2e-test resourceVersion: "19377" uid: 2d4b9978-964d-419e-b6c2-4693a1625360 kind: Event lastTimestamp: "2026-04-23T18:13:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:13:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:15Z" name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5.18a90efa3e181cae namespace: kserve-ci-e2e-test resourceVersion: "19437" uid: f94d9a2b-8ece-4827-8476-58236753f02c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5 namespace: kserve-ci-e2e-test resourceVersion: "19377" uid: 2d4b9978-964d-419e-b6c2-4693a1625360 kind: Event lastTimestamp: "2026-04-23T18:13:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:13:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:15Z" name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5.18a90efa3e249dbf namespace: kserve-ci-e2e-test resourceVersion: "19438" uid: f2c2a622-b925-4d76-9d8d-d68b9aacb340 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5 namespace: kserve-ci-e2e-test resourceVersion: "19377" uid: 2d4b9978-964d-419e-b6c2-4693a1625360 kind: Event lastTimestamp: "2026-04-23T18:13:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:13:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:15Z" name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5.18a90efa41217285 namespace: kserve-ci-e2e-test resourceVersion: "19442" uid: 6c1aff89-7ff8-4c38-b26b-880f510f45ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5 namespace: kserve-ci-e2e-test resourceVersion: "19377" uid: 2d4b9978-964d-419e-b6c2-4693a1625360 kind: Event lastTimestamp: "2026-04-23T18:13:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:13:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:15Z" name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5.18a90efa419230d8 namespace: kserve-ci-e2e-test resourceVersion: "19443" uid: ad4eacea-0573-468a-bd44-8de6b9788b03 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T18:13:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5 namespace: kserve-ci-e2e-test resourceVersion: "19377" uid: 2d4b9978-964d-419e-b6c2-4693a1625360 kind: Event lastTimestamp: "2026-04-23T18:14:33Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:13:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:33Z" name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5.18a90efab287ad23 namespace: kserve-ci-e2e-test resourceVersion: "20077" uid: 8aaddbaf-e33a-4922-85f7-2a1d0a99ee20 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5 namespace: kserve-ci-e2e-test resourceVersion: "19377" uid: 2d4b9978-964d-419e-b6c2-4693a1625360 kind: Event lastTimestamp: "2026-04-23T18:14:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:31Z" name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5.18a90f0bd508337d namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: 89ba6620-4124-48f7-9a84-8c078e0ff13b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5 namespace: kserve-ci-e2e-test resourceVersion: "19377" uid: 2d4b9978-964d-419e-b6c2-4693a1625360 kind: Event lastTimestamp: "2026-04-23T18:14:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:31Z" name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5.18a90f0bd508b28e namespace: kserve-ci-e2e-test resourceVersion: "20030" uid: 2ed007ea-985f-4e8f-b510-793a12bb03fd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5 namespace: kserve-ci-e2e-test resourceVersion: "19377" uid: 2d4b9978-964d-419e-b6c2-4693a1625360 kind: Event lastTimestamp: "2026-04-23T18:14:33Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:14:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:33Z" name: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5.18a90f0c649e59d8 namespace: kserve-ci-e2e-test resourceVersion: "20076" uid: 84263d44-e38b-4ea6-b5f5-4124f17a1c98 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-0f5e3-predictor-675fcf7fdc namespace: kserve-ci-e2e-test resourceVersion: "19365" uid: 79557f81-10e2-4bf2-a9ff-d171568c7b66 kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: 'Created pod: raw-sklearn-0f5e3-predictor-675fcf7fdc-zq4l5' metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:11Z" name: raw-sklearn-0f5e3-predictor-675fcf7fdc.18a90ef926c93d6d namespace: kserve-ci-e2e-test resourceVersion: "19378" uid: 8c1a1bbf-bbca-4e3e-901c-12be59739fcf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-0f5e3-predictor namespace: kserve-ci-e2e-test resourceVersion: "19363" uid: aa6469db-530d-4f7c-813c-050be9f04dab kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: Scaled up replica set raw-sklearn-0f5e3-predictor-675fcf7fdc from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:11Z" name: raw-sklearn-0f5e3-predictor.18a90ef9243186eb namespace: kserve-ci-e2e-test resourceVersion: "19367" uid: c2fa3f2e-46a3-4594-ac04-5ea7f6b8786c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:13:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-0f5e3-predictor namespace: kserve-ci-e2e-test resourceVersion: "19364" uid: c0f3cd7e-92d4-4f30-825c-8b4b0a456d86 kind: Event lastTimestamp: "2026-04-23T18:13:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:41Z" name: raw-sklearn-0f5e3-predictor.18a90efca31fbe87 namespace: kserve-ci-e2e-test resourceVersion: "19628" uid: 81b65393-5700-42a9-8e88-e6d924824c65 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:13:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-0f5e3-predictor namespace: kserve-ci-e2e-test resourceVersion: "19364" uid: c0f3cd7e-92d4-4f30-825c-8b4b0a456d86 kind: Event lastTimestamp: "2026-04-23T18:13:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:41Z" name: raw-sklearn-0f5e3-predictor.18a90efca3b337fc namespace: kserve-ci-e2e-test resourceVersion: "19630" uid: 55467e1d-c35a-4757-8e78-2bfb93e7be02 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:13:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-0f5e3-predictor namespace: kserve-ci-e2e-test resourceVersion: "19525" uid: c0f3cd7e-92d4-4f30-825c-8b4b0a456d86 kind: Event lastTimestamp: "2026-04-23T18:14:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:26Z" name: raw-sklearn-0f5e3-predictor.18a90f03a1be6245 namespace: kserve-ci-e2e-test resourceVersion: "19951" uid: c2b928c3-8eb1-4fb0-9421-9c5c648b99b5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:13:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-0f5e3-predictor namespace: kserve-ci-e2e-test resourceVersion: "19525" uid: c0f3cd7e-92d4-4f30-825c-8b4b0a456d86 kind: Event lastTimestamp: "2026-04-23T18:14:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:26Z" name: raw-sklearn-0f5e3-predictor.18a90f03a22ff0b0 namespace: kserve-ci-e2e-test resourceVersion: "19952" uid: d7934738-5c49-4643-8785-c158df5a92c5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-0f5e3 namespace: kserve-ci-e2e-test resourceVersion: "19352" uid: fb381441-889c-4c06-b8e9-dcf006f4fb2f kind: Event lastTimestamp: "2026-04-23T18:14:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:14:24Z" name: raw-sklearn-0f5e3.18a90ef9240efbe6 namespace: kserve-ci-e2e-test resourceVersion: "19937" uid: bf01356a-f2dc-4ccf-bc8d-18ae934909ec reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-0f5e3 namespace: kserve-ci-e2e-test resourceVersion: "19352" uid: fb381441-889c-4c06-b8e9-dcf006f4fb2f kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: 'Failed to update status for InferenceService "raw-sklearn-0f5e3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-0f5e3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:11Z" name: raw-sklearn-0f5e3.18a90ef92460cca7 namespace: kserve-ci-e2e-test resourceVersion: "19391" uid: f0471f26-bcd9-413c-9609-afa447ddb8aa reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-0f5e3 namespace: kserve-ci-e2e-test resourceVersion: "19373" uid: fb381441-889c-4c06-b8e9-dcf006f4fb2f kind: Event lastTimestamp: "2026-04-23T18:13:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-0f5e3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:11Z" name: raw-sklearn-0f5e3.18a90ef935656da9 namespace: kserve-ci-e2e-test resourceVersion: "19396" uid: b26bde9f-4bc3-44d3-9c9c-7a487802261d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-0f5e3 namespace: kserve-ci-e2e-test resourceVersion: "19935" uid: fb381441-889c-4c06-b8e9-dcf006f4fb2f kind: Event lastTimestamp: "2026-04-23T18:14:23Z" message: InferenceService [raw-sklearn-0f5e3] is Ready metadata: creationTimestamp: "2026-04-23T18:14:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:14:23Z" name: raw-sklearn-0f5e3.18a90f0a1681ecb7 namespace: kserve-ci-e2e-test resourceVersion: "19936" uid: 93a790bc-0b8c-42be-9d88-d120261b4c0f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:14:31.490244Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm namespace: kserve-ci-e2e-test resourceVersion: "20028" uid: 9a3b6fb6-7d51-41b7-bd08-85493b58d372 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm to ip-10-0-132-102.ec2.internal metadata: creationTimestamp: "2026-04-23T18:14:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:14:31Z" name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm.18a90f0bd66335e0 namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: 4eac7e71-373e-4fc9-90ac-2524a2bce7c3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-774bdb58b-lw5wg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:31Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm namespace: kserve-ci-e2e-test resourceVersion: "20041" uid: 9a3b6fb6-7d51-41b7-bd08-85493b58d372 kind: Event lastTimestamp: "2026-04-23T18:14:31Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:14:31Z" name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm.18a90f0beefa72d6 namespace: kserve-ci-e2e-test resourceVersion: "20050" uid: f40ba7dc-82db-4d8a-aa93-60d54e3165d1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 9a3b6fb6-7d51-41b7-bd08-85493b58d372 kind: Event lastTimestamp: "2026-04-23T18:14:31Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:48bcc0e7da25ba7560a82df7216c0ad464b268d851c665b85b9195165f27b89d" already present on machine metadata: creationTimestamp: "2026-04-23T18:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:31Z" name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm.18a90f0bf0bc9244 namespace: kserve-ci-e2e-test resourceVersion: "20053" uid: 93eaf47a-9c6b-4116-9ef8-a250e455b072 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 9a3b6fb6-7d51-41b7-bd08-85493b58d372 kind: Event lastTimestamp: "2026-04-23T18:14:32Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T18:14:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:32Z" name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm.18a90f0bf50d6eb3 namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: ce7911e9-969b-4faa-bb0f-fc9e05a609e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 9a3b6fb6-7d51-41b7-bd08-85493b58d372 kind: Event lastTimestamp: "2026-04-23T18:14:32Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T18:14:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:32Z" name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm.18a90f0bf5c1fd20 namespace: kserve-ci-e2e-test resourceVersion: "20055" uid: 4e5c9e5a-c417-4369-9d48-121d869e3cc1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 9a3b6fb6-7d51-41b7-bd08-85493b58d372 kind: Event lastTimestamp: "2026-04-23T18:14:36Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1417" already present on machine metadata: creationTimestamp: "2026-04-23T18:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:36Z" name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm.18a90f0ce8794818 namespace: kserve-ci-e2e-test resourceVersion: "20095" uid: 39761201-f2b8-48fe-82a2-4b982c080294 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 9a3b6fb6-7d51-41b7-bd08-85493b58d372 kind: Event lastTimestamp: "2026-04-23T18:14:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:36Z" name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm.18a90f0cebd9788d namespace: kserve-ci-e2e-test resourceVersion: "20098" uid: 985b796c-76ff-4913-a445-d5ba0440c5de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 9a3b6fb6-7d51-41b7-bd08-85493b58d372 kind: Event lastTimestamp: "2026-04-23T18:14:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:36Z" name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm.18a90f0cec5eb334 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: 1cabfa6f-b371-4979-9b9b-0401cf464ad1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 9a3b6fb6-7d51-41b7-bd08-85493b58d372 kind: Event lastTimestamp: "2026-04-23T18:14:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:36Z" name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm.18a90f0cec6b843f namespace: kserve-ci-e2e-test resourceVersion: "20102" uid: be9f4005-8456-4304-a52e-edb4d4d02ec6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 9a3b6fb6-7d51-41b7-bd08-85493b58d372 kind: Event lastTimestamp: "2026-04-23T18:14:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:36Z" name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm.18a90f0cef7a9ba0 namespace: kserve-ci-e2e-test resourceVersion: "20104" uid: b3db3630-f913-4cd3-8d4d-8631bdc1b724 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 9a3b6fb6-7d51-41b7-bd08-85493b58d372 kind: Event lastTimestamp: "2026-04-23T18:14:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:36Z" name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm.18a90f0ceff18157 namespace: kserve-ci-e2e-test resourceVersion: "20105" uid: 921eb94b-102e-4fe8-8a94-dc56da4847c1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T18:14:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 9a3b6fb6-7d51-41b7-bd08-85493b58d372 kind: Event lastTimestamp: "2026-04-23T18:15:54Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:15:54Z" name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm.18a90f0d6026dd49 namespace: kserve-ci-e2e-test resourceVersion: "20716" uid: c7a7e9ec-4ea4-442f-bb7f-d85c04769686 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:15:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 9a3b6fb6-7d51-41b7-bd08-85493b58d372 kind: Event lastTimestamp: "2026-04-23T18:15:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:15:51Z" name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm.18a90f1e7efd8a7b namespace: kserve-ci-e2e-test resourceVersion: "20665" uid: 514036e4-f8b6-4d9f-9982-c16ac2fa847e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:15:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 9a3b6fb6-7d51-41b7-bd08-85493b58d372 kind: Event lastTimestamp: "2026-04-23T18:15:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:15:51Z" name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm.18a90f1e7f01d39e namespace: kserve-ci-e2e-test resourceVersion: "20666" uid: 31e0d98e-6ec1-4c88-8437-2da3b29c0a50 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:15:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 9a3b6fb6-7d51-41b7-bd08-85493b58d372 kind: Event lastTimestamp: "2026-04-23T18:15:54Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:15:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:15:54Z" name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm.18a90f1f1242e98a namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: 878eec57-b3d0-453b-a8bf-ce4089198bb8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-102.ec2.internal source: component: kubelet host: ip-10-0-132-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7 namespace: kserve-ci-e2e-test resourceVersion: "20006" uid: a8ad8740-49b2-4847-9f54-ddfbf95c050c kind: Event lastTimestamp: "2026-04-23T18:14:31Z" message: 'Created pod: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7-mpgsm' metadata: creationTimestamp: "2026-04-23T18:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:31Z" name: raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7.18a90f0bd5f2245d namespace: kserve-ci-e2e-test resourceVersion: "20032" uid: 77db7bf6-466d-4221-bfb0-7b615132d485 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-5ccec-predictor namespace: kserve-ci-e2e-test resourceVersion: "20002" uid: 7d80ed93-e1c6-4b22-9267-0c9fbc072137 kind: Event lastTimestamp: "2026-04-23T18:14:31Z" message: Scaled up replica set raw-sklearn-runtime-5ccec-predictor-6dbd6f74c7 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:31Z" name: raw-sklearn-runtime-5ccec-predictor.18a90f0bd15b1273 namespace: kserve-ci-e2e-test resourceVersion: "20012" uid: e670a80e-9975-4ba5-80ac-06ee1ed3b51b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:14:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-5ccec-predictor namespace: kserve-ci-e2e-test resourceVersion: "20005" uid: 24963e90-6056-4afe-bea7-df44297448d3 kind: Event lastTimestamp: "2026-04-23T18:15:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:15:01Z" name: raw-sklearn-runtime-5ccec-predictor.18a90f0f500c72c9 namespace: kserve-ci-e2e-test resourceVersion: "20289" uid: ed567d62-f51f-4277-a873-68ff603ee6eb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:14:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-5ccec-predictor namespace: kserve-ci-e2e-test resourceVersion: "20005" uid: 24963e90-6056-4afe-bea7-df44297448d3 kind: Event lastTimestamp: "2026-04-23T18:15:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:15:01Z" name: raw-sklearn-runtime-5ccec-predictor.18a90f0f508170d6 namespace: kserve-ci-e2e-test resourceVersion: "20290" uid: 211276af-1547-4edf-ac9b-4ba9b492b766 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:15:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-5ccec-predictor namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: 24963e90-6056-4afe-bea7-df44297448d3 kind: Event lastTimestamp: "2026-04-23T18:15:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:15:46Z" name: raw-sklearn-runtime-5ccec-predictor.18a90f164ebb894d namespace: kserve-ci-e2e-test resourceVersion: "20610" uid: e972ace1-0762-47e1-858c-e6f1fffb378e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:15:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-5ccec-predictor namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: 24963e90-6056-4afe-bea7-df44297448d3 kind: Event lastTimestamp: "2026-04-23T18:15:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:15:46Z" name: raw-sklearn-runtime-5ccec-predictor.18a90f164f2ab17d namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: 374d0466-961d-4f47-9ab3-aeb30432e513 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-23T18:14:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-5ccec namespace: kserve-ci-e2e-test resourceVersion: "19995" uid: e4f15d01-d24d-487a-ae8a-aae9b545e25c kind: Event lastTimestamp: "2026-04-23T18:15:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:15:44Z" name: raw-sklearn-runtime-5ccec.18a90f0bd1358f3a namespace: kserve-ci-e2e-test resourceVersion: "20593" uid: 53b89899-eb6b-4f6b-b975-56dfcf0c54d1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-5ccec namespace: kserve-ci-e2e-test resourceVersion: "20014" uid: e4f15d01-d24d-487a-ae8a-aae9b545e25c kind: Event lastTimestamp: "2026-04-23T18:14:31Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-5ccec": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-5ccec": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:14:31Z" name: raw-sklearn-runtime-5ccec.18a90f0bd6a97d07 namespace: kserve-ci-e2e-test resourceVersion: "20039" uid: f3e98e3a-4143-4e31-bfea-f24bd5e927c2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:15:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-5ccec namespace: kserve-ci-e2e-test resourceVersion: "20590" uid: e4f15d01-d24d-487a-ae8a-aae9b545e25c kind: Event lastTimestamp: "2026-04-23T18:15:44Z" message: InferenceService [raw-sklearn-runtime-5ccec] is Ready metadata: creationTimestamp: "2026-04-23T18:15:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:15:44Z" name: raw-sklearn-runtime-5ccec.18a90f1cc3f0e36b namespace: kserve-ci-e2e-test resourceVersion: "20592" uid: 044b1b0d-38da-46b1-b1f4-6b72a7ad111a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "20862"