--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:20:55.210900Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-531344-predictor-684755d79b-ghbr5 namespace: kserve-ci-e2e-test resourceVersion: "21142" uid: 75d473c2-c721-4e08-aed3-c70db8a7b969 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-531344-predictor-684755d79b-ghbr5 to ip-10-0-129-35.ec2.internal metadata: creationTimestamp: "2026-07-20T10:20:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:20:55Z" name: isvc-init-fail-531344-predictor-684755d79b-ghbr5.18c3f8459b4eb1ab namespace: kserve-ci-e2e-test resourceVersion: "21149" uid: ebfd8a04-9ae7-43f6-ba83-ac0e058ae297 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cc9bcc47d-zhtvs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:55Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-531344-predictor-684755d79b-ghbr5 namespace: kserve-ci-e2e-test resourceVersion: "21145" uid: 75d473c2-c721-4e08-aed3-c70db8a7b969 kind: Event lastTimestamp: "2026-07-20T10:20:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-init-fail-531344-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:20:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:55Z" name: isvc-init-fail-531344-predictor-684755d79b-ghbr5.18c3f845aa771906 namespace: kserve-ci-e2e-test resourceVersion: "21156" uid: 17a520fa-77a6-4d7b-9f9c-5f0b4dee309b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-531344-predictor-684755d79b-ghbr5 namespace: kserve-ci-e2e-test resourceVersion: "21153" uid: 75d473c2-c721-4e08-aed3-c70db8a7b969 kind: Event lastTimestamp: "2026-07-20T10:20:56Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:20:56Z" name: isvc-init-fail-531344-predictor-684755d79b-ghbr5.18c3f845d91cf03f namespace: kserve-ci-e2e-test resourceVersion: "21170" uid: de1a3006-7de1-4929-8b82-e27f307e8dc6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:20:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-531344-predictor-684755d79b-ghbr5 namespace: kserve-ci-e2e-test resourceVersion: "21145" uid: 75d473c2-c721-4e08-aed3-c70db8a7b969 kind: Event lastTimestamp: "2026-07-20T10:21:02Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:038cd29f3e031cdfd5fa54ba896bbecc69cd255eda6b470c2f7bce5282357bb8" already present on machine metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:02Z" name: isvc-init-fail-531344-predictor-684755d79b-ghbr5.18c3f845dac45b9e namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: c4e4c9a7-399a-49d8-8960-9b5186851fb6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:20:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-531344-predictor-684755d79b-ghbr5 namespace: kserve-ci-e2e-test resourceVersion: "21145" uid: 75d473c2-c721-4e08-aed3-c70db8a7b969 kind: Event lastTimestamp: "2026-07-20T10:21:02Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:02Z" name: isvc-init-fail-531344-predictor-684755d79b-ghbr5.18c3f845dd77328a namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: b1d96007-5c1d-41b6-9105-08d4e70fb6f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:20:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-531344-predictor-684755d79b-ghbr5 namespace: kserve-ci-e2e-test resourceVersion: "21145" uid: 75d473c2-c721-4e08-aed3-c70db8a7b969 kind: Event lastTimestamp: "2026-07-20T10:21:02Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:02Z" name: isvc-init-fail-531344-predictor-684755d79b-ghbr5.18c3f845ddf07caf namespace: kserve-ci-e2e-test resourceVersion: "21225" uid: 0f43626a-69b1-4acf-a998-d0e71c6fae22 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-531344-predictor-684755d79b-ghbr5 namespace: kserve-ci-e2e-test resourceVersion: "21145" uid: 75d473c2-c721-4e08-aed3-c70db8a7b969 kind: Event lastTimestamp: "2026-07-20T10:21:05Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-07-20T10:21:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:05Z" name: isvc-init-fail-531344-predictor-684755d79b-ghbr5.18c3f847ef8037be namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: ae0f6f82-2978-4726-9b2f-3cd62d64dc11 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-531344-predictor-684755d79b namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: ec446751-0584-48c9-9aaf-4221214411ee kind: Event lastTimestamp: "2026-07-20T10:20:55Z" message: 'Created pod: isvc-init-fail-531344-predictor-684755d79b-ghbr5' metadata: creationTimestamp: "2026-07-20T10:20:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:55Z" name: isvc-init-fail-531344-predictor-684755d79b.18c3f8459ac949db namespace: kserve-ci-e2e-test resourceVersion: "21143" uid: 14fcc0bb-0faa-4e0f-a632-398d00c3f6e0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-531344-predictor namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 6d17a72b-4ec2-4989-97f6-41dbf25e41e8 kind: Event lastTimestamp: "2026-07-20T10:20:55Z" message: Scaled up replica set isvc-init-fail-531344-predictor-684755d79b from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:20:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:55Z" name: isvc-init-fail-531344-predictor.18c3f8459769dcfe namespace: kserve-ci-e2e-test resourceVersion: "21135" uid: e3b1e852-2687-4dfb-ab9b-29c7850a45ea reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-20T10:20:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-531344 namespace: kserve-ci-e2e-test resourceVersion: "21125" uid: e0d7076b-1fcf-4526-9818-199015f3ebbf kind: Event lastTimestamp: "2026-07-20T10:21:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:20:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:21:02Z" name: isvc-init-fail-531344.18c3f8459793ac0a namespace: kserve-ci-e2e-test resourceVersion: "21227" uid: 5f880206-31c3-4134-846f-591af3067596 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-531344 namespace: kserve-ci-e2e-test resourceVersion: "21125" uid: e0d7076b-1fcf-4526-9818-199015f3ebbf kind: Event lastTimestamp: "2026-07-20T10:20:55Z" message: 'Failed to update status for InferenceService "isvc-init-fail-531344": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-531344": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:20:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:20:55Z" name: isvc-init-fail-531344.18c3f84597fb0b72 namespace: kserve-ci-e2e-test resourceVersion: "21138" uid: 0888e9c1-25fb-46ee-b43f-e47ad2fb9b50 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-531344 namespace: kserve-ci-e2e-test resourceVersion: "21125" uid: e0d7076b-1fcf-4526-9818-199015f3ebbf kind: Event lastTimestamp: "2026-07-20T10:20:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-531344": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:20:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:20:55Z" name: isvc-init-fail-531344.18c3f84597fb5026 namespace: kserve-ci-e2e-test resourceVersion: "21140" uid: dd5572b3-0fa1-4c25-b0b1-d6e04545539b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:14:06.005288Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg namespace: kserve-ci-e2e-test resourceVersion: "17872" uid: aef7e726-3136-4c35-9eed-567c438d3ec8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg to ip-10-0-129-35.ec2.internal metadata: creationTimestamp: "2026-07-20T10:14:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:14:06Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg.18c3f7e654c0943d namespace: kserve-ci-e2e-test resourceVersion: "17879" uid: 345e0b9c-c6c7-4c3d-8dba-add18dec314b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cc9bcc47d-zhtvs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:14:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg namespace: kserve-ci-e2e-test resourceVersion: "17884" uid: aef7e726-3136-4c35-9eed-567c438d3ec8 kind: Event lastTimestamp: "2026-07-20T10:14:06Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:14:06Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg.18c3f7e66eeb0c77 namespace: kserve-ci-e2e-test resourceVersion: "17892" uid: 944db496-67d0-4c1c-9c03-be2146b2b5cb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:14:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: aef7e726-3136-4c35-9eed-567c438d3ec8 kind: Event lastTimestamp: "2026-07-20T10:14:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:038cd29f3e031cdfd5fa54ba896bbecc69cd255eda6b470c2f7bce5282357bb8" already present on machine metadata: creationTimestamp: "2026-07-20T10:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:14:06Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg.18c3f7e671268bd7 namespace: kserve-ci-e2e-test resourceVersion: "17894" uid: b1251ef9-eb28-4b1a-a8cc-35c34ecb5694 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:14:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: aef7e726-3136-4c35-9eed-567c438d3ec8 kind: Event lastTimestamp: "2026-07-20T10:14:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-20T10:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:14:06Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg.18c3f7e673ff35b6 namespace: kserve-ci-e2e-test resourceVersion: "17895" uid: ea7edb2a-c358-44c5-897a-533b3541c1f7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:14:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: aef7e726-3136-4c35-9eed-567c438d3ec8 kind: Event lastTimestamp: "2026-07-20T10:14:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-20T10:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:14:06Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg.18c3f7e6747621c4 namespace: kserve-ci-e2e-test resourceVersion: "17896" uid: fe32a45a-3fd8-45f3-ba53-d0b3a4f2b52c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: aef7e726-3136-4c35-9eed-567c438d3ec8 kind: Event lastTimestamp: "2026-07-20T10:14:11Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:14:11Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg.18c3f7e787df00dc namespace: kserve-ci-e2e-test resourceVersion: "17935" uid: 1809c8e8-b67e-4243-bd51-3ff4aecac838 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: aef7e726-3136-4c35-9eed-567c438d3ec8 kind: Event lastTimestamp: "2026-07-20T10:14:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:14:11Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg.18c3f7e78b10b2b0 namespace: kserve-ci-e2e-test resourceVersion: "17937" uid: 1ba44ae3-78d9-40d9-b4a2-fd5e6dc25428 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: aef7e726-3136-4c35-9eed-567c438d3ec8 kind: Event lastTimestamp: "2026-07-20T10:14:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:14:11Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg.18c3f7e78b8cf78b namespace: kserve-ci-e2e-test resourceVersion: "17938" uid: b994544d-4558-426e-8970-9f0377c1cc7b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: aef7e726-3136-4c35-9eed-567c438d3ec8 kind: Event lastTimestamp: "2026-07-20T10:14:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:14:11Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg.18c3f7e78b9833e4 namespace: kserve-ci-e2e-test resourceVersion: "17941" uid: 11988772-a3ea-4298-92ce-ebf3e348993e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: aef7e726-3136-4c35-9eed-567c438d3ec8 kind: Event lastTimestamp: "2026-07-20T10:14:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:14:11Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg.18c3f7e78f762b6b namespace: kserve-ci-e2e-test resourceVersion: "17943" uid: 14a4129d-9a57-4324-92f4-5cad0fb11042 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: aef7e726-3136-4c35-9eed-567c438d3ec8 kind: Event lastTimestamp: "2026-07-20T10:14:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:14:11Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg.18c3f7e78ff28abc namespace: kserve-ci-e2e-test resourceVersion: "17944" uid: a016a8e0-f6d6-4d86-ae4f-5ee72df5debd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: aef7e726-3136-4c35-9eed-567c438d3ec8 kind: Event lastTimestamp: "2026-07-20T10:14:11Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:07d9be4ee48ab8e3f13b5e400054db7129a4dfd20edc4167e0ae24bd38c38bfa" already present on machine metadata: creationTimestamp: "2026-07-20T10:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:14:11Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg.18c3f7e78ffded3e namespace: kserve-ci-e2e-test resourceVersion: "17945" uid: cf79b55a-ea64-48d7-9bc0-2c8f5c4c6019 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: aef7e726-3136-4c35-9eed-567c438d3ec8 kind: Event lastTimestamp: "2026-07-20T10:14:11Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-20T10:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:14:11Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg.18c3f7e7933d2682 namespace: kserve-ci-e2e-test resourceVersion: "17947" uid: e2d7a9e5-07f1-44ac-8d50-e109ddde4d02 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: aef7e726-3136-4c35-9eed-567c438d3ec8 kind: Event lastTimestamp: "2026-07-20T10:14:11Z" message: Started container agent metadata: creationTimestamp: "2026-07-20T10:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:14:11Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg.18c3f7e7941a3e7f namespace: kserve-ci-e2e-test resourceVersion: "17948" uid: 2c788f47-6303-4ec5-8337-1d945e0f6717 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-20T10:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: aef7e726-3136-4c35-9eed-567c438d3ec8 kind: Event lastTimestamp: "2026-07-20T10:15:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:15:49Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg.18c3f7e7c3f1cde4 namespace: kserve-ci-e2e-test resourceVersion: "18759" uid: d6280817-e295-4e8b-b4d7-ddfa347068d4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-20T10:14:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: aef7e726-3136-4c35-9eed-567c438d3ec8 kind: Event lastTimestamp: "2026-07-20T10:15:49Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-20T10:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:15:49Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg.18c3f7e7ffc01f4a namespace: kserve-ci-e2e-test resourceVersion: "18760" uid: 9db2520f-1581-43ee-979c-2119b60e97ba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:15:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: aef7e726-3136-4c35-9eed-567c438d3ec8 kind: Event lastTimestamp: "2026-07-20T10:15:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:15:41Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg.18c3f7fc862d70a5 namespace: kserve-ci-e2e-test resourceVersion: "18653" uid: c9a18307-cf20-463f-bd01-5f74705b6fcf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:15:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: aef7e726-3136-4c35-9eed-567c438d3ec8 kind: Event lastTimestamp: "2026-07-20T10:15:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:15:41Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg.18c3f7fc862f44aa namespace: kserve-ci-e2e-test resourceVersion: "18658" uid: d5970555-4752-48a6-b34f-178801704c8c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:15:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: aef7e726-3136-4c35-9eed-567c438d3ec8 kind: Event lastTimestamp: "2026-07-20T10:15:41Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-20T10:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:15:41Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg.18c3f7fc8630af5a namespace: kserve-ci-e2e-test resourceVersion: "18661" uid: 79df6d25-c8bd-43b6-a4a2-ac78fe6b59a5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:15:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: aef7e726-3136-4c35-9eed-567c438d3ec8 kind: Event lastTimestamp: "2026-07-20T10:15:59Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:15:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:15:59Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg.18c3f7fd2ffb9f21 namespace: kserve-ci-e2e-test resourceVersion: "18843" uid: cbcb3ef3-9f27-4c48-bdc6-b6e498236e0f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:14:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-22c6a-predictor-7c89f597df namespace: kserve-ci-e2e-test resourceVersion: "17862" uid: f94ac7c9-7474-4fe1-a2dd-864728100cf7 kind: Event lastTimestamp: "2026-07-20T10:14:05Z" message: 'Created pod: isvc-logger-raw-22c6a-predictor-7c89f597df-4f4mg' metadata: creationTimestamp: "2026-07-20T10:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:14:06Z" name: isvc-logger-raw-22c6a-predictor-7c89f597df.18c3f7e65458416b namespace: kserve-ci-e2e-test resourceVersion: "17876" uid: 8d360395-03c9-4e76-b8a2-8e35e71582db reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:14:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-22c6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17860" uid: 2f1461f6-0406-418a-8fa3-67ae0287d23b kind: Event lastTimestamp: "2026-07-20T10:14:05Z" message: Scaled up replica set isvc-logger-raw-22c6a-predictor-7c89f597df from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:14:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:14:05Z" name: isvc-logger-raw-22c6a-predictor.18c3f7e64dd0f6f2 namespace: kserve-ci-e2e-test resourceVersion: "17864" uid: b7efec1c-db77-4a50-81ce-5bc1d40d637e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:14:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-22c6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17861" uid: 97891e1a-50bd-490f-85a8-f3497738c496 kind: Event lastTimestamp: "2026-07-20T10:14:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:14:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:14:35Z" name: isvc-logger-raw-22c6a-predictor.18c3f7e9cc2b91db namespace: kserve-ci-e2e-test resourceVersion: "18134" uid: 9312a959-93cb-4c7c-ad6e-ec1d0fb2c5b1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:14:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-22c6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17861" uid: 97891e1a-50bd-490f-85a8-f3497738c496 kind: Event lastTimestamp: "2026-07-20T10:14:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:14:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:14:35Z" name: isvc-logger-raw-22c6a-predictor.18c3f7e9ccc136e2 namespace: kserve-ci-e2e-test resourceVersion: "18135" uid: 2e9a5ee6-d391-46bb-8a7f-350f882d10e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:14:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-22c6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "18033" uid: 97891e1a-50bd-490f-85a8-f3497738c496 kind: Event lastTimestamp: "2026-07-20T10:15:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:15:35Z" name: isvc-logger-raw-22c6a-predictor.18c3f7f0ca51cd8d namespace: kserve-ci-e2e-test resourceVersion: "18570" uid: 48a28e71-0bde-4e3d-9a45-43d204560eda reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:14:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-22c6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "18033" uid: 97891e1a-50bd-490f-85a8-f3497738c496 kind: Event lastTimestamp: "2026-07-20T10:15:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:15:35Z" name: isvc-logger-raw-22c6a-predictor.18c3f7f0cae858b3 namespace: kserve-ci-e2e-test resourceVersion: "18571" uid: 22d96bd4-6b12-436b-b6c7-ad3ca0c53315 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-20T10:14:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-22c6a namespace: kserve-ci-e2e-test resourceVersion: "17853" uid: 770c1efd-e0a0-476c-8bae-495ae79c3bfa kind: Event lastTimestamp: "2026-07-20T10:15:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:14:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:15:29Z" name: isvc-logger-raw-22c6a.18c3f7e64dcc71eb namespace: kserve-ci-e2e-test resourceVersion: "18522" uid: 84e0a9ac-4e04-4f88-8172-2da0313c0b44 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:14:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-22c6a namespace: kserve-ci-e2e-test resourceVersion: "17853" uid: 770c1efd-e0a0-476c-8bae-495ae79c3bfa kind: Event lastTimestamp: "2026-07-20T10:14:05Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-22c6a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-22c6a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:14:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:14:05Z" name: isvc-logger-raw-22c6a.18c3f7e64eea2099 namespace: kserve-ci-e2e-test resourceVersion: "17868" uid: 87ee6735-e7b6-40c2-9259-33102ac4d29a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:15:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-22c6a namespace: kserve-ci-e2e-test resourceVersion: "18519" uid: 770c1efd-e0a0-476c-8bae-495ae79c3bfa kind: Event lastTimestamp: "2026-07-20T10:15:29Z" message: InferenceService [isvc-logger-raw-22c6a] is Ready metadata: creationTimestamp: "2026-07-20T10:15:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:15:29Z" name: isvc-logger-raw-22c6a.18c3f7f9b8f1d980 namespace: kserve-ci-e2e-test resourceVersion: "18521" uid: 5adc466e-77c5-45b7-bca0-afe9eb358574 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:18:46.605292Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr namespace: kserve-ci-e2e-test resourceVersion: "20059" uid: 966b0087-3b89-4a89-9a1d-11b3c7a28d0d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-cefa95-predictor-5866df9cfc-2vppr to ip-10-0-129-35.ec2.internal metadata: creationTimestamp: "2026-07-20T10:18:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:18:46Z" name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr.18c3f827a9d0d7c6 namespace: kserve-ci-e2e-test resourceVersion: "20063" uid: 66e05114-8ded-49c5-8022-cc98d87a38a0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cc9bcc47d-zhtvs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:18:47Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr namespace: kserve-ci-e2e-test resourceVersion: "20068" uid: 966b0087-3b89-4a89-9a1d-11b3c7a28d0d kind: Event lastTimestamp: "2026-07-20T10:18:47Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:18:47Z" name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr.18c3f827c3bcdc07 namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: f44a8715-8c4f-482d-a99c-89b52e30b000 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:18:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 966b0087-3b89-4a89-9a1d-11b3c7a28d0d kind: Event lastTimestamp: "2026-07-20T10:18:47Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:038cd29f3e031cdfd5fa54ba896bbecc69cd255eda6b470c2f7bce5282357bb8" already present on machine metadata: creationTimestamp: "2026-07-20T10:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:18:47Z" name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr.18c3f827c5ef0587 namespace: kserve-ci-e2e-test resourceVersion: "20078" uid: 234bfc75-3378-4fa0-a265-c911fe1c88df reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:18:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 966b0087-3b89-4a89-9a1d-11b3c7a28d0d kind: Event lastTimestamp: "2026-07-20T10:18:47Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-20T10:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:18:47Z" name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr.18c3f827c8ddc95e namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: 73a7cd9f-da2d-4577-8e7d-9aa8d67df90c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:18:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 966b0087-3b89-4a89-9a1d-11b3c7a28d0d kind: Event lastTimestamp: "2026-07-20T10:18:47Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-20T10:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:18:47Z" name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr.18c3f827c95483b7 namespace: kserve-ci-e2e-test resourceVersion: "20080" uid: 8aff1640-2657-40b5-a3fd-3df0ff67fd69 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:18:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 966b0087-3b89-4a89-9a1d-11b3c7a28d0d kind: Event lastTimestamp: "2026-07-20T10:18:51Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:18:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:18:51Z" name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr.18c3f828cbb6b657 namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: 52002efb-5c2d-41d9-8ae5-e3655956ba80 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:18:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 966b0087-3b89-4a89-9a1d-11b3c7a28d0d kind: Event lastTimestamp: "2026-07-20T10:18:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:18:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:18:51Z" name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr.18c3f828ceb3d43c namespace: kserve-ci-e2e-test resourceVersion: "20120" uid: 86401c61-0fc9-41df-874b-970a7b0e8989 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:18:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 966b0087-3b89-4a89-9a1d-11b3c7a28d0d kind: Event lastTimestamp: "2026-07-20T10:18:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:18:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:18:51Z" name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr.18c3f828cf2e3f28 namespace: kserve-ci-e2e-test resourceVersion: "20121" uid: c82f37a3-3260-4619-b469-ee91ca497a61 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:18:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 966b0087-3b89-4a89-9a1d-11b3c7a28d0d kind: Event lastTimestamp: "2026-07-20T10:18:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:18:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:18:51Z" name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr.18c3f828cf3a1b23 namespace: kserve-ci-e2e-test resourceVersion: "20124" uid: fc174825-d125-4951-b7ec-4c1cd51c0e8a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:18:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 966b0087-3b89-4a89-9a1d-11b3c7a28d0d kind: Event lastTimestamp: "2026-07-20T10:18:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:18:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:18:51Z" name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr.18c3f828d22bc06d namespace: kserve-ci-e2e-test resourceVersion: "20126" uid: 0572287b-f83b-443f-947f-cde54288bd7d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:18:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 966b0087-3b89-4a89-9a1d-11b3c7a28d0d kind: Event lastTimestamp: "2026-07-20T10:18:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:18:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:18:51Z" name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr.18c3f828d2acd963 namespace: kserve-ci-e2e-test resourceVersion: "20128" uid: 73dc412c-bfa4-41b0-87cf-1581aae324a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-20T10:18:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 966b0087-3b89-4a89-9a1d-11b3c7a28d0d kind: Event lastTimestamp: "2026-07-20T10:20:54Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:54Z" name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr.18c3f82943d91891 namespace: kserve-ci-e2e-test resourceVersion: "21111" uid: fc9eccef-a10a-4989-8ba8-8bf50b2bf411 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 966b0087-3b89-4a89-9a1d-11b3c7a28d0d kind: Event lastTimestamp: "2026-07-20T10:20:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:20:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:49Z" name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr.18c3f84460b663ba namespace: kserve-ci-e2e-test resourceVersion: "21062" uid: 96a3f4d0-b5a3-4fe4-849a-b014d11dc0cd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 966b0087-3b89-4a89-9a1d-11b3c7a28d0d kind: Event lastTimestamp: "2026-07-20T10:20:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:20:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:49Z" name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr.18c3f84460b7a9a7 namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: e6c68822-1882-4350-a77a-a4f5d998f578 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 966b0087-3b89-4a89-9a1d-11b3c7a28d0d kind: Event lastTimestamp: "2026-07-20T10:20:54Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:54Z" name: isvc-primary-cefa95-predictor-5866df9cfc-2vppr.18c3f8457044fc6f namespace: kserve-ci-e2e-test resourceVersion: "21112" uid: ba226afb-7bae-4434-9e62-6b78a46f3a34 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:18:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-cefa95-predictor-5866df9cfc namespace: kserve-ci-e2e-test resourceVersion: "20050" uid: 37358cef-7604-4c74-9054-44dbc3cbdfeb kind: Event lastTimestamp: "2026-07-20T10:18:46Z" message: 'Created pod: isvc-primary-cefa95-predictor-5866df9cfc-2vppr' metadata: creationTimestamp: "2026-07-20T10:18:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:18:46Z" name: isvc-primary-cefa95-predictor-5866df9cfc.18c3f827a90e09fc namespace: kserve-ci-e2e-test resourceVersion: "20060" uid: 2228053f-637b-409d-afee-25f61bf9894b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:18:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-cefa95-predictor namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 942359a5-b207-4f4f-b73e-f7eea5363cb7 kind: Event lastTimestamp: "2026-07-20T10:18:46Z" message: Scaled up replica set isvc-primary-cefa95-predictor-5866df9cfc from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:18:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:18:46Z" name: isvc-primary-cefa95-predictor.18c3f827a455a9f9 namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: f7f533a9-7221-4c91-8f7e-50ec77b5f404 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:19:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-cefa95-predictor namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: c9683c98-f536-4cbf-9689-c9536bf86407 kind: Event lastTimestamp: "2026-07-20T10:19:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:19:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:19:31Z" name: isvc-primary-cefa95-predictor.18c3f82b23812302 namespace: kserve-ci-e2e-test resourceVersion: "20422" uid: ec501455-c487-4f8b-9580-4822779224eb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:19:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-cefa95-predictor namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: c9683c98-f536-4cbf-9689-c9536bf86407 kind: Event lastTimestamp: "2026-07-20T10:19:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:19:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:19:31Z" name: isvc-primary-cefa95-predictor.18c3f82b243f1ce8 namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: 748c306f-82ad-462b-944a-bc4a1501b965 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:19:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-cefa95-predictor namespace: kserve-ci-e2e-test resourceVersion: "20210" uid: c9683c98-f536-4cbf-9689-c9536bf86407 kind: Event lastTimestamp: "2026-07-20T10:20:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:46Z" name: isvc-primary-cefa95-predictor.18c3f835a18ccfa7 namespace: kserve-ci-e2e-test resourceVersion: "21017" uid: 7dcfed69-5007-413e-b47d-1425b1de9421 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:19:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-cefa95-predictor namespace: kserve-ci-e2e-test resourceVersion: "20210" uid: c9683c98-f536-4cbf-9689-c9536bf86407 kind: Event lastTimestamp: "2026-07-20T10:20:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:46Z" name: isvc-primary-cefa95-predictor.18c3f835a217cad8 namespace: kserve-ci-e2e-test resourceVersion: "21018" uid: 797faf73-b3d9-4f97-a2ef-9f4870954741 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-20T10:18:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-cefa95 namespace: kserve-ci-e2e-test resourceVersion: "20041" uid: 8937e3b5-db56-43ea-be91-b81f6308d1fa kind: Event lastTimestamp: "2026-07-20T10:20:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:18:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:20:14Z" name: isvc-primary-cefa95.18c3f827a453b050 namespace: kserve-ci-e2e-test resourceVersion: "20723" uid: eb3cbd4d-610d-4fb8-a50f-14e28833e5bd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:18:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-cefa95 namespace: kserve-ci-e2e-test resourceVersion: "20041" uid: 8937e3b5-db56-43ea-be91-b81f6308d1fa kind: Event lastTimestamp: "2026-07-20T10:18:46Z" message: 'Failed to update status for InferenceService "isvc-primary-cefa95": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-cefa95": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:18:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:18:46Z" name: isvc-primary-cefa95.18c3f827a5077bb6 namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: c4a6436f-d165-4e96-9052-5d2ecadf009a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:18:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-cefa95 namespace: kserve-ci-e2e-test resourceVersion: "20041" uid: 8937e3b5-db56-43ea-be91-b81f6308d1fa kind: Event lastTimestamp: "2026-07-20T10:18:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-cefa95": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:18:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:18:46Z" name: isvc-primary-cefa95.18c3f827a507cfd1 namespace: kserve-ci-e2e-test resourceVersion: "20055" uid: e987b577-dd71-4605-8010-b9c19e668cb5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-cefa95 namespace: kserve-ci-e2e-test resourceVersion: "20721" uid: 8937e3b5-db56-43ea-be91-b81f6308d1fa kind: Event lastTimestamp: "2026-07-20T10:20:14Z" message: InferenceService [isvc-primary-cefa95] is Ready metadata: creationTimestamp: "2026-07-20T10:20:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:20:14Z" name: isvc-primary-cefa95.18c3f83c25b84a24 namespace: kserve-ci-e2e-test resourceVersion: "20722" uid: 053d204b-0fad-4453-a23c-2efd69888cd7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:08:09.997740Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14665" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p to ip-10-0-129-35.ec2.internal metadata: creationTimestamp: "2026-07-20T10:08:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:08:09Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f793710c8792 namespace: kserve-ci-e2e-test resourceVersion: "14672" uid: f22ebfa8-7d61-4393-8f3e-7f20209e5bea reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cc9bcc47d-zhtvs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14675" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:08:10Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:08:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:08:10Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f7938b0c6519 namespace: kserve-ci-e2e-test resourceVersion: "14683" uid: 5b231a82-0f86-4841-8680-8bba313f2ada reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:08:10Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:038cd29f3e031cdfd5fa54ba896bbecc69cd255eda6b470c2f7bce5282357bb8" metadata: creationTimestamp: "2026-07-20T10:08:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:08:10Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f7938cdbf393 namespace: kserve-ci-e2e-test resourceVersion: "14685" uid: 19147e68-1532-4ace-be68-79838364536f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:08:13Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:038cd29f3e031cdfd5fa54ba896bbecc69cd255eda6b470c2f7bce5282357bb8" in 3.096s (3.096s including waiting). Image size: 300593107 bytes.' metadata: creationTimestamp: "2026-07-20T10:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:08:13Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f7944567ea8f namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: eea44ce6-a79c-4e63-9268-4236f1434511 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:08:13Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-20T10:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:08:13Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f79448a027f5 namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: c2bbb859-12d6-4d4f-8148-71f601bbac17 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:08:13Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-20T10:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:08:13Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f794491b656d namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: 22507f90-3ed2-4834-8512-6b9bc666794d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:08:18Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1771" metadata: creationTimestamp: "2026-07-20T10:08:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:08:18Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f79566dc0be4 namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: 36bc2cca-35d6-41c3-8233-928c2f6520b7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:08:32Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1771" in 14.073s (14.073s including waiting). Image size: 1555845555 bytes.' metadata: creationTimestamp: "2026-07-20T10:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:08:32Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f798adb27263 namespace: kserve-ci-e2e-test resourceVersion: "14847" uid: f70b5dfc-744f-4e12-92ae-1b9982c1822b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:08:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:08:32Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f798b1e360af namespace: kserve-ci-e2e-test resourceVersion: "14848" uid: cf74dc59-3fd0-4959-a745-65dc7216d461 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:08:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:08:32Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f798b2603b79 namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: 57a28276-3602-4e6b-a01e-5b2b00eebe31 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:08:32Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-20T10:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:08:32Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f798b26edaf7 namespace: kserve-ci-e2e-test resourceVersion: "14850" uid: c0de3b91-e44e-461f-98a1-1d753180eb65 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:08:34Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.272s (2.272s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-20T10:08:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:08:34Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f79939e4685f namespace: kserve-ci-e2e-test resourceVersion: "14871" uid: 9c4200d7-f49f-497b-be00-85ad51402413 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:08:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:08:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:08:34Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f7993ce9aeaf namespace: kserve-ci-e2e-test resourceVersion: "14873" uid: 9a03d73f-0989-42b9-a42d-9ec2ea23b657 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:08:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:08:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:08:34Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f7993d633b66 namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: decce370-caed-490a-a953-e60fd09321d8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:08:34Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:07d9be4ee48ab8e3f13b5e400054db7129a4dfd20edc4167e0ae24bd38c38bfa" metadata: creationTimestamp: "2026-07-20T10:08:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:08:34Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f7993d71cdbb namespace: kserve-ci-e2e-test resourceVersion: "14875" uid: a08babba-ebfc-4cb0-9a3b-1c8c2e707d1e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:08:37Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:07d9be4ee48ab8e3f13b5e400054db7129a4dfd20edc4167e0ae24bd38c38bfa" in 2.171s (2.171s including waiting). Image size: 240172661 bytes.' metadata: creationTimestamp: "2026-07-20T10:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:08:37Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f799bed9e50f namespace: kserve-ci-e2e-test resourceVersion: "14893" uid: 665534e7-3ed9-4f8c-9be6-333806b2f1f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:08:37Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-20T10:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:08:37Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f799c2bc3b5c namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: b4a95d97-5cba-4440-ac60-e118ebf000ce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:08:37Z" message: Started container agent metadata: creationTimestamp: "2026-07-20T10:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:08:37Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f799c33b4494 namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 46134608-edc6-4e5c-ac4c-b64c3dfea1b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-20T10:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:10:23Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:23Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f799db1a5495 namespace: kserve-ci-e2e-test resourceVersion: "15812" uid: c48d5648-46f5-4f64-ae37-f72ae2b439e5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-20T10:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:10:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-20T10:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:13Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f799db2567e3 namespace: kserve-ci-e2e-test resourceVersion: "15728" uid: 8e3a6304-eb80-4337-bd4c-6af8b33a0889 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:10:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:10:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:04Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f7ae36f89dfb namespace: kserve-ci-e2e-test resourceVersion: "15562" uid: d74ad121-8634-4a8c-aab1-5d20f0b61e0c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:10:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:10:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:04Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f7ae36f9d5ff namespace: kserve-ci-e2e-test resourceVersion: "15565" uid: bfc3a7f3-a710-4b39-885b-02273189e6c5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:10:04Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-20T10:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:05Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f7ae36f9e5a3 namespace: kserve-ci-e2e-test resourceVersion: "15569" uid: d401c76f-115b-4420-b0b1-397de55f4fa2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:10:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 32dba575-515b-4185-bb26-4e4a59694f1d kind: Event lastTimestamp: "2026-07-20T10:10:23Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:10:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:23Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p.18c3f7af0b4e86d3 namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: 2a6ec83e-d027-4d9d-adeb-dcbf845df418 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: dd326825-91be-4d00-af79-93f4db828bd1 kind: Event lastTimestamp: "2026-07-20T10:08:09Z" message: 'Created pod: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b-8c74p' metadata: creationTimestamp: "2026-07-20T10:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:08:09Z" name: isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b.18c3f7937035ac53 namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: 5783dee7-ad85-4cd8-95ed-e1be801ebc9b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-a9709-predictor namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 5896bbc9-1ded-499c-a956-07fc53efd359 kind: Event lastTimestamp: "2026-07-20T10:08:09Z" message: Scaled up replica set isvc-raw-sklearn-batcher-a9709-predictor-d685dcf9b from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:08:09Z" name: isvc-raw-sklearn-batcher-a9709-predictor.18c3f7936682508a namespace: kserve-ci-e2e-test resourceVersion: "14654" uid: b9ecfa7c-0500-47ad-b123-7739a7dae47a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:08:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-a9709-predictor namespace: kserve-ci-e2e-test resourceVersion: "14660" uid: 49938817-9eb0-479a-995c-e89792bc28ab kind: Event lastTimestamp: "2026-07-20T10:09:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:09:10Z" name: isvc-raw-sklearn-batcher-a9709-predictor.18c3f796ecbc7de6 namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: 0743c331-6db0-4143-ac18-36a3289ab2f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:08:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-a9709-predictor namespace: kserve-ci-e2e-test resourceVersion: "14660" uid: 49938817-9eb0-479a-995c-e89792bc28ab kind: Event lastTimestamp: "2026-07-20T10:09:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:09:10Z" name: isvc-raw-sklearn-batcher-a9709-predictor.18c3f796ed816886 namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: 282523b4-30b4-487b-b320-6ab9cb2c00e2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:08:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-a9709-predictor namespace: kserve-ci-e2e-test resourceVersion: "14794" uid: 49938817-9eb0-479a-995c-e89792bc28ab kind: Event lastTimestamp: "2026-07-20T10:09:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:09:55Z" name: isvc-raw-sklearn-batcher-a9709-predictor.18c3f79deb6ffc68 namespace: kserve-ci-e2e-test resourceVersion: "15471" uid: a1a8779e-ba15-4f5e-aa24-012d93003c00 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:08:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-a9709-predictor namespace: kserve-ci-e2e-test resourceVersion: "14794" uid: 49938817-9eb0-479a-995c-e89792bc28ab kind: Event lastTimestamp: "2026-07-20T10:09:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:09:55Z" name: isvc-raw-sklearn-batcher-a9709-predictor.18c3f79dec0d114f namespace: kserve-ci-e2e-test resourceVersion: "15472" uid: 2645455d-9f26-4953-b31d-1270dc95a573 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-20T10:08:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-a9709 namespace: kserve-ci-e2e-test resourceVersion: "14645" uid: 8322a14f-5065-4363-bf3c-88cd8df85feb kind: Event lastTimestamp: "2026-07-20T10:09:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:09:53Z" name: isvc-raw-sklearn-batcher-a9709.18c3f7936c7a6739 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: fbac188b-4b35-426c-9d7c-b42d56d4fa99 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-a9709 namespace: kserve-ci-e2e-test resourceVersion: "14645" uid: 8322a14f-5065-4363-bf3c-88cd8df85feb kind: Event lastTimestamp: "2026-07-20T10:08:09Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-a9709": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-a9709": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:08:09Z" name: isvc-raw-sklearn-batcher-a9709.18c3f7936d238bfe namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: b05e31f1-d865-42fb-88d0-3a0df01cb94f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:08:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-a9709 namespace: kserve-ci-e2e-test resourceVersion: "14645" uid: 8322a14f-5065-4363-bf3c-88cd8df85feb kind: Event lastTimestamp: "2026-07-20T10:08:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-a9709": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:08:09Z" name: isvc-raw-sklearn-batcher-a9709.18c3f7936d23fc54 namespace: kserve-ci-e2e-test resourceVersion: "14663" uid: 2df66922-bc57-4b52-9615-8d5b73ae5f30 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-a9709 namespace: kserve-ci-e2e-test resourceVersion: "15459" uid: 8322a14f-5065-4363-bf3c-88cd8df85feb kind: Event lastTimestamp: "2026-07-20T10:09:53Z" message: InferenceService [isvc-raw-sklearn-batcher-a9709] is Ready metadata: creationTimestamp: "2026-07-20T10:09:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:09:53Z" name: isvc-raw-sklearn-batcher-a9709.18c3f7ab9a0e1276 namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: bdb4ce54-2933-4f13-bc23-8d026aef9e1b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:20:31.960810Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-cefa95-predictor-664c5956d6-h5cvl namespace: kserve-ci-e2e-test resourceVersion: "20873" uid: da29b45a-e485-4fa7-b8cc-d4c004062292 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-cefa95-predictor-664c5956d6-h5cvl to ip-10-0-129-35.ec2.internal metadata: creationTimestamp: "2026-07-20T10:20:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:20:31Z" name: isvc-secondary-cefa95-predictor-664c5956d6-h5cvl.18c3f840317e7f76 namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: 6fda6087-585a-4abb-9862-8fe929f7ecf1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cc9bcc47d-zhtvs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:32Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-cefa95-predictor-664c5956d6-h5cvl namespace: kserve-ci-e2e-test resourceVersion: "20887" uid: da29b45a-e485-4fa7-b8cc-d4c004062292 kind: Event lastTimestamp: "2026-07-20T10:20:32Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:20:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:20:32Z" name: isvc-secondary-cefa95-predictor-664c5956d6-h5cvl.18c3f8404b8d2c35 namespace: kserve-ci-e2e-test resourceVersion: "20892" uid: 1fbf65b6-1ccd-44a2-b9e5-c95941ca9d6d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:20:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-cefa95-predictor-664c5956d6-h5cvl namespace: kserve-ci-e2e-test resourceVersion: "20875" uid: da29b45a-e485-4fa7-b8cc-d4c004062292 kind: Event lastTimestamp: "2026-07-20T10:20:38Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:038cd29f3e031cdfd5fa54ba896bbecc69cd255eda6b470c2f7bce5282357bb8" already present on machine metadata: creationTimestamp: "2026-07-20T10:20:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:38Z" name: isvc-secondary-cefa95-predictor-664c5956d6-h5cvl.18c3f8404d1621dd namespace: kserve-ci-e2e-test resourceVersion: "20947" uid: 7f382b39-59c6-483b-bc39-ec1efaf88cc9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:20:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-cefa95-predictor-664c5956d6-h5cvl namespace: kserve-ci-e2e-test resourceVersion: "20875" uid: da29b45a-e485-4fa7-b8cc-d4c004062292 kind: Event lastTimestamp: "2026-07-20T10:20:39Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-20T10:20:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:39Z" name: isvc-secondary-cefa95-predictor-664c5956d6-h5cvl.18c3f84050a8f742 namespace: kserve-ci-e2e-test resourceVersion: "20951" uid: 2253149c-ee93-4740-9994-cd94d3c90356 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:20:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-cefa95-predictor-664c5956d6-h5cvl namespace: kserve-ci-e2e-test resourceVersion: "20875" uid: da29b45a-e485-4fa7-b8cc-d4c004062292 kind: Event lastTimestamp: "2026-07-20T10:20:39Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-20T10:20:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:39Z" name: isvc-secondary-cefa95-predictor-664c5956d6-h5cvl.18c3f8405141a403 namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: c7bb7f15-2632-47a1-94b4-6d19b3ed2563 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-cefa95-predictor-664c5956d6-h5cvl namespace: kserve-ci-e2e-test resourceVersion: "20875" uid: da29b45a-e485-4fa7-b8cc-d4c004062292 kind: Event lastTimestamp: "2026-07-20T10:20:44Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-cefa95-predictor-664c5956d6-h5cvl_kserve-ci-e2e-test(da29b45a-e485-4fa7-b8cc-d4c004062292) metadata: creationTimestamp: "2026-07-20T10:20:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:45Z" name: isvc-secondary-cefa95-predictor-664c5956d6-h5cvl.18c3f8433aa47bc9 namespace: kserve-ci-e2e-test resourceVersion: "21000" uid: 590af7e5-6669-48b9-802d-68518209e22c reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-cefa95-predictor-664c5956d6 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 11c480ec-bdad-4288-b435-e312a0c258b9 kind: Event lastTimestamp: "2026-07-20T10:20:31Z" message: 'Created pod: isvc-secondary-cefa95-predictor-664c5956d6-h5cvl' metadata: creationTimestamp: "2026-07-20T10:20:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:31Z" name: isvc-secondary-cefa95-predictor-664c5956d6.18c3f84030e69431 namespace: kserve-ci-e2e-test resourceVersion: "20874" uid: 4f1628b8-0a72-40ba-ad07-756c59f5cf4d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-cefa95-predictor namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: be91c964-35c3-44d9-afdf-a5da902a6544 kind: Event lastTimestamp: "2026-07-20T10:20:31Z" message: Scaled up replica set isvc-secondary-cefa95-predictor-664c5956d6 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:20:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:31Z" name: isvc-secondary-cefa95-predictor.18c3f8402bb51737 namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: 107ed77f-b4ad-422b-aab8-ff9c62e3937c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-cefa95-predictor namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 26218c3b-3d0c-4966-9f6f-529fd01bbdce kind: Event lastTimestamp: "2026-07-20T10:20:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:20:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:46Z" name: isvc-secondary-cefa95-predictor.18c3f843aa8e2ae4 namespace: kserve-ci-e2e-test resourceVersion: "21021" uid: c62c914e-4129-4963-86cb-4653ed2d0279 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-cefa95-predictor namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 26218c3b-3d0c-4966-9f6f-529fd01bbdce kind: Event lastTimestamp: "2026-07-20T10:20:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:20:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:46Z" name: isvc-secondary-cefa95-predictor.18c3f843ab2579b6 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: d3210d1e-65b5-4c32-b0fa-14b2bc9820b1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-20T10:20:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-cefa95 namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: 4b63fd31-6ebb-44d8-a00a-08938f3c3f17 kind: Event lastTimestamp: "2026-07-20T10:20:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:20:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:20:45Z" name: isvc-secondary-cefa95.18c3f8402bb2313a namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: e02f4e5c-34e9-4f32-b6fa-17f0b4d25597 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:20:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-cefa95 namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: 4b63fd31-6ebb-44d8-a00a-08938f3c3f17 kind: Event lastTimestamp: "2026-07-20T10:20:31Z" message: 'Failed to update status for InferenceService "isvc-secondary-cefa95": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-cefa95": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:20:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:20:31Z" name: isvc-secondary-cefa95.18c3f8402c6b3a6c namespace: kserve-ci-e2e-test resourceVersion: "20884" uid: 5f5936c5-f63c-48b8-ba8e-ea1ae1cf32b5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:20:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-cefa95 namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: 4b63fd31-6ebb-44d8-a00a-08938f3c3f17 kind: Event lastTimestamp: "2026-07-20T10:20:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-cefa95": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:20:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:20:31Z" name: isvc-secondary-cefa95.18c3f8402c6b7c74 namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: 0fb47cf6-3436-40fe-8eff-5cc67abdd4c3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:10:05.085088Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88 namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: e3166688-b723-458a-818f-b367cbeb37fa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88 to ip-10-0-129-35.ec2.internal metadata: creationTimestamp: "2026-07-20T10:10:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:10:05Z" name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88.18c3f7ae3cca21bb namespace: kserve-ci-e2e-test resourceVersion: "15591" uid: f01f962e-b716-486b-a336-242bc510dfd0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cc9bcc47d-zhtvs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88 namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: e3166688-b723-458a-818f-b367cbeb37fa kind: Event lastTimestamp: "2026-07-20T10:10:05Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:10:05Z" name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88.18c3f7ae57725ca9 namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: 22c9bc83-bad9-437b-a774-3cbd333d0cce reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: e3166688-b723-458a-818f-b367cbeb37fa kind: Event lastTimestamp: "2026-07-20T10:10:05Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:038cd29f3e031cdfd5fa54ba896bbecc69cd255eda6b470c2f7bce5282357bb8" already present on machine metadata: creationTimestamp: "2026-07-20T10:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:05Z" name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88.18c3f7ae593a4c87 namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 905e0de0-9168-4506-9105-c31c41c0595d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: e3166688-b723-458a-818f-b367cbeb37fa kind: Event lastTimestamp: "2026-07-20T10:10:05Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-20T10:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:05Z" name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88.18c3f7ae5c3e32f2 namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: c3155fb0-a9af-4391-b3e3-6553c8027c34 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: e3166688-b723-458a-818f-b367cbeb37fa kind: Event lastTimestamp: "2026-07-20T10:10:05Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-20T10:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:05Z" name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88.18c3f7ae5cd3b783 namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: 891a7622-b939-4e0a-8aa3-432daf912807 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: e3166688-b723-458a-818f-b367cbeb37fa kind: Event lastTimestamp: "2026-07-20T10:10:09Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:10:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:09Z" name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88.18c3f7af607c831b namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: b8cae2eb-ad05-4923-b54a-12d5199a57a4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: e3166688-b723-458a-818f-b367cbeb37fa kind: Event lastTimestamp: "2026-07-20T10:10:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:10Z" name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88.18c3f7af6376aac2 namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: 56a70525-3dd3-4840-ad64-fe75ce06ec7b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: e3166688-b723-458a-818f-b367cbeb37fa kind: Event lastTimestamp: "2026-07-20T10:10:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:10Z" name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88.18c3f7af63f13693 namespace: kserve-ci-e2e-test resourceVersion: "15689" uid: 95522df8-894f-4e25-8bd4-3d63bbd91f75 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: e3166688-b723-458a-818f-b367cbeb37fa kind: Event lastTimestamp: "2026-07-20T10:10:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:10Z" name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88.18c3f7af63fe8ffa namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: eec6b951-074e-48cf-b472-b47997eee9e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: e3166688-b723-458a-818f-b367cbeb37fa kind: Event lastTimestamp: "2026-07-20T10:10:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:10Z" name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88.18c3f7af6af1570e namespace: kserve-ci-e2e-test resourceVersion: "15694" uid: 41910d58-6b2c-44af-9e62-3e91ef45b5e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: e3166688-b723-458a-818f-b367cbeb37fa kind: Event lastTimestamp: "2026-07-20T10:10:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:10Z" name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88.18c3f7af6b6c9dde namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: 4af8dcc0-c855-46e1-9d43-c150e038e577 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-20T10:10:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: e3166688-b723-458a-818f-b367cbeb37fa kind: Event lastTimestamp: "2026-07-20T10:11:28Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:10:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:28Z" name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88.18c3f7afd8c15594 namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: 442f185b-a283-42a1-955a-b4377cb5d817 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: e3166688-b723-458a-818f-b367cbeb37fa kind: Event lastTimestamp: "2026-07-20T10:11:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:55Z" name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88.18c3f7c7eb6a431a namespace: kserve-ci-e2e-test resourceVersion: "16597" uid: fee73df7-a0a3-4fb7-9e3c-b0ae960da8ef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: e3166688-b723-458a-818f-b367cbeb37fa kind: Event lastTimestamp: "2026-07-20T10:11:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:55Z" name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88.18c3f7c7eb6aad1e namespace: kserve-ci-e2e-test resourceVersion: "16600" uid: df7c486b-b9f1-4cc9-b80d-3dab2d11de6a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: e3166688-b723-458a-818f-b367cbeb37fa kind: Event lastTimestamp: "2026-07-20T10:11:58Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:11:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:58Z" name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88.18c3f7c8875f6d3e namespace: kserve-ci-e2e-test resourceVersion: "16718" uid: 41f22336-e6c6-46d5-a3c5-2502a1668b61 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: 18ec62e6-276f-4c36-a4b6-7548f20b6da6 kind: Event lastTimestamp: "2026-07-20T10:10:05Z" message: 'Created pod: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88-8pd88' metadata: creationTimestamp: "2026-07-20T10:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:05Z" name: isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88.18c3f7ae3c5f8ef6 namespace: kserve-ci-e2e-test resourceVersion: "15583" uid: 482af699-17db-404c-8b71-3c1fd6e5e7e9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-d73f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15563" uid: b93d40dd-0d68-4c5b-926f-3ca2c786fd28 kind: Event lastTimestamp: "2026-07-20T10:10:05Z" message: Scaled up replica set isvc-sklearn-graph-raw-d73f8-predictor-76dbbfdc88 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:05Z" name: isvc-sklearn-graph-raw-d73f8-predictor.18c3f7ae3842b2ad namespace: kserve-ci-e2e-test resourceVersion: "15571" uid: 3cdef15b-ba66-4ac6-aa63-878546636086 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:10:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-d73f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: 0f91154f-3f26-4d53-a00f-7a663cf7b843 kind: Event lastTimestamp: "2026-07-20T10:10:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:35Z" name: isvc-sklearn-graph-raw-d73f8-predictor.18c3f7b1b71d08ce namespace: kserve-ci-e2e-test resourceVersion: "15897" uid: c9601064-6c03-4f7d-9c85-8fd9d545aa04 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:10:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-d73f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: 0f91154f-3f26-4d53-a00f-7a663cf7b843 kind: Event lastTimestamp: "2026-07-20T10:10:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:35Z" name: isvc-sklearn-graph-raw-d73f8-predictor.18c3f7b1b79f959f namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: 6853f22b-78fd-4062-a5b9-3b4fd6870082 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:10:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-d73f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15785" uid: 0f91154f-3f26-4d53-a00f-7a663cf7b843 kind: Event lastTimestamp: "2026-07-20T10:11:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:50Z" name: isvc-sklearn-graph-raw-d73f8-predictor.18c3f7b8b5be15a8 namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: c38edf95-e3a1-4abd-87f9-36914851c2d5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:10:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-d73f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15785" uid: 0f91154f-3f26-4d53-a00f-7a663cf7b843 kind: Event lastTimestamp: "2026-07-20T10:11:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:50Z" name: isvc-sklearn-graph-raw-d73f8-predictor.18c3f7b8b640017e namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: c2848e2b-d723-4f4f-955d-aeb6520b2b58 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-20T10:10:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-d73f8 namespace: kserve-ci-e2e-test resourceVersion: "15541" uid: 9a4492be-a00d-4b3d-bba2-3a0af15a0f85 kind: Event lastTimestamp: "2026-07-20T10:11:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:11:33Z" name: isvc-sklearn-graph-raw-d73f8.18c3f7ae38400e0b namespace: kserve-ci-e2e-test resourceVersion: "16345" uid: 61aee965-3f0d-4b7c-b3a0-38fb5131a32c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-d73f8 namespace: kserve-ci-e2e-test resourceVersion: "15541" uid: 9a4492be-a00d-4b3d-bba2-3a0af15a0f85 kind: Event lastTimestamp: "2026-07-20T10:10:05Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-d73f8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-d73f8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:10:05Z" name: isvc-sklearn-graph-raw-d73f8.18c3f7ae38c804a1 namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: b49e5553-9665-45c5-976e-1d72139fe5f9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-d73f8 namespace: kserve-ci-e2e-test resourceVersion: "16343" uid: 9a4492be-a00d-4b3d-bba2-3a0af15a0f85 kind: Event lastTimestamp: "2026-07-20T10:11:33Z" message: InferenceService [isvc-sklearn-graph-raw-d73f8] is Ready metadata: creationTimestamp: "2026-07-20T10:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:11:33Z" name: isvc-sklearn-graph-raw-d73f8.18c3f7c2d970f302 namespace: kserve-ci-e2e-test resourceVersion: "16344" uid: c6278c4b-ec39-4e8e-bec1-907e4bf00f65 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:11:55.467677Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz namespace: kserve-ci-e2e-test resourceVersion: "16607" uid: 60b15f70-56fa-479a-a8a8-8ae06c26890a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz to ip-10-0-129-35.ec2.internal metadata: creationTimestamp: "2026-07-20T10:11:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:11:55Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz.18c3f7c7f01abc9b namespace: kserve-ci-e2e-test resourceVersion: "16616" uid: 9f2d57bc-bb11-4dd1-96e6-575b1cd3ee0f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cc9bcc47d-zhtvs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:55Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: 60b15f70-56fa-479a-a8a8-8ae06c26890a kind: Event lastTimestamp: "2026-07-20T10:11:55Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:11:55Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz.18c3f7c80aa68d38 namespace: kserve-ci-e2e-test resourceVersion: "16678" uid: 8f8172ec-1860-4a09-88b1-dfa3d0385ee5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 60b15f70-56fa-479a-a8a8-8ae06c26890a kind: Event lastTimestamp: "2026-07-20T10:11:55Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:038cd29f3e031cdfd5fa54ba896bbecc69cd255eda6b470c2f7bce5282357bb8" already present on machine metadata: creationTimestamp: "2026-07-20T10:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:55Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz.18c3f7c80c9bf52f namespace: kserve-ci-e2e-test resourceVersion: "16681" uid: d72225c6-3103-46e3-8ffd-3a51d035865e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 60b15f70-56fa-479a-a8a8-8ae06c26890a kind: Event lastTimestamp: "2026-07-20T10:11:55Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-20T10:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:55Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz.18c3f7c80f5e81a6 namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: 692ba086-9e01-46ae-9853-8b5e768b9647 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 60b15f70-56fa-479a-a8a8-8ae06c26890a kind: Event lastTimestamp: "2026-07-20T10:11:56Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-20T10:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:56Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz.18c3f7c80fd899de namespace: kserve-ci-e2e-test resourceVersion: "16686" uid: cb6befb5-5c40-4a0d-b38c-af253dcd1a6e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 60b15f70-56fa-479a-a8a8-8ae06c26890a kind: Event lastTimestamp: "2026-07-20T10:12:00Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:12:00Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz.18c3f7c91e1626fe namespace: kserve-ci-e2e-test resourceVersion: "16737" uid: 8c3a56f3-e50b-49f8-94d3-c3df8f46f565 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 60b15f70-56fa-479a-a8a8-8ae06c26890a kind: Event lastTimestamp: "2026-07-20T10:12:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:12:00Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz.18c3f7c9223a2d28 namespace: kserve-ci-e2e-test resourceVersion: "16743" uid: 501be6a7-1636-4f36-8304-dee9f5d81697 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 60b15f70-56fa-479a-a8a8-8ae06c26890a kind: Event lastTimestamp: "2026-07-20T10:12:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:12:00Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz.18c3f7c922be1a3e namespace: kserve-ci-e2e-test resourceVersion: "16744" uid: a3ba1f24-b390-47c0-b4f1-59672ff20e5b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 60b15f70-56fa-479a-a8a8-8ae06c26890a kind: Event lastTimestamp: "2026-07-20T10:12:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:12:00Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz.18c3f7c922cc2c63 namespace: kserve-ci-e2e-test resourceVersion: "16745" uid: f7ce9cbe-f612-48ad-9adc-b3316d6eb48a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 60b15f70-56fa-479a-a8a8-8ae06c26890a kind: Event lastTimestamp: "2026-07-20T10:12:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:12:00Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz.18c3f7c92777a141 namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: a7aeb2c4-73da-4fe9-8135-6f16b520d279 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 60b15f70-56fa-479a-a8a8-8ae06c26890a kind: Event lastTimestamp: "2026-07-20T10:12:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:12:00Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz.18c3f7c927f285fd namespace: kserve-ci-e2e-test resourceVersion: "16749" uid: 02ab98ef-f15e-420f-a310-20d91c8bbdc5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-20T10:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 60b15f70-56fa-479a-a8a8-8ae06c26890a kind: Event lastTimestamp: "2026-07-20T10:13:12Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:13:12Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz.18c3f7c95ac365a1 namespace: kserve-ci-e2e-test resourceVersion: "17297" uid: 2515df74-ba52-4f82-8141-684f31a492ac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 60b15f70-56fa-479a-a8a8-8ae06c26890a kind: Event lastTimestamp: "2026-07-20T10:13:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:13:46Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz.18c3f7e1b836cd56 namespace: kserve-ci-e2e-test resourceVersion: "17657" uid: f2bda209-3d03-4d7b-accb-2a5b5d116aa1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 60b15f70-56fa-479a-a8a8-8ae06c26890a kind: Event lastTimestamp: "2026-07-20T10:13:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:13:46Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz.18c3f7e1b8379fe7 namespace: kserve-ci-e2e-test resourceVersion: "17658" uid: db926014-6a15-4148-955c-846194b6b897 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 60b15f70-56fa-479a-a8a8-8ae06c26890a kind: Event lastTimestamp: "2026-07-20T10:13:47Z" message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp 10.133.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:13:47Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz.18c3f7e209360db7 namespace: kserve-ci-e2e-test resourceVersion: "17694" uid: eedeaa33-5955-4089-96e1-631e33d36b67 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848 namespace: kserve-ci-e2e-test resourceVersion: "16581" uid: e9cd1d5f-3e8e-4b3c-ae67-c6f60927594a kind: Event lastTimestamp: "2026-07-20T10:11:55Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848-k2pmz' metadata: creationTimestamp: "2026-07-20T10:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:55Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848.18c3f7c7ef7fc9a7 namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: 1f849720-4925-42ce-87d1-a8569471e1c3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16576" uid: 157430ef-55d1-44f1-b71d-b48c79cd7387 kind: Event lastTimestamp: "2026-07-20T10:11:55Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-0bd3d-predictor-64d6667848 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:55Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor.18c3f7c7e8a8f297 namespace: kserve-ci-e2e-test resourceVersion: "16585" uid: ad211362-2837-47fe-8e22-f71e404b9ff6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:12:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16577" uid: 5338ef40-89ed-4964-8511-42ee64e0130d kind: Event lastTimestamp: "2026-07-20T10:12:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:12:25Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor.18c3f7cb671b3521 namespace: kserve-ci-e2e-test resourceVersion: "16950" uid: 449fc637-0a30-42f7-9494-8d6bf9550656 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:12:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16577" uid: 5338ef40-89ed-4964-8511-42ee64e0130d kind: Event lastTimestamp: "2026-07-20T10:12:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:12:25Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor.18c3f7cb6788a263 namespace: kserve-ci-e2e-test resourceVersion: "16951" uid: 2a66429b-0d8e-4996-9b8f-d1ce95c791d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:12:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16837" uid: 5338ef40-89ed-4964-8511-42ee64e0130d kind: Event lastTimestamp: "2026-07-20T10:13:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:12:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:13:40Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor.18c3f7d2659bf670 namespace: kserve-ci-e2e-test resourceVersion: "17545" uid: 232f2401-9d2b-4474-a7a6-af5f7186f39f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:12:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16837" uid: 5338ef40-89ed-4964-8511-42ee64e0130d kind: Event lastTimestamp: "2026-07-20T10:13:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:12:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:13:40Z" name: isvc-sklearn-graph-raw-hpa-0bd3d-predictor.18c3f7d2660d9353 namespace: kserve-ci-e2e-test resourceVersion: "17546" uid: f67ea552-327e-4459-9741-8396dd2e047c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-20T10:11:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-0bd3d namespace: kserve-ci-e2e-test resourceVersion: "16569" uid: 672569d3-4838-4fd8-96de-3e26ec8301f1 kind: Event lastTimestamp: "2026-07-20T10:13:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:13:22Z" name: isvc-sklearn-graph-raw-hpa-0bd3d.18c3f7c7e8b426f3 namespace: kserve-ci-e2e-test resourceVersion: "17387" uid: 656d1c52-3ded-4295-bdc6-98c10e714797 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-0bd3d namespace: kserve-ci-e2e-test resourceVersion: "17383" uid: 672569d3-4838-4fd8-96de-3e26ec8301f1 kind: Event lastTimestamp: "2026-07-20T10:13:22Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-0bd3d] is Ready metadata: creationTimestamp: "2026-07-20T10:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:13:22Z" name: isvc-sklearn-graph-raw-hpa-0bd3d.18c3f7dc39b32587 namespace: kserve-ci-e2e-test resourceVersion: "17385" uid: 6ec84da7-c6a3-4bde-9b18-5bce5da61d7f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:15:41.342899Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 60889480-945c-43a5-bf90-accfb84f6997 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n to ip-10-0-129-35.ec2.internal metadata: creationTimestamp: "2026-07-20T10:15:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:15:41Z" name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n.18c3f7fc875126c2 namespace: kserve-ci-e2e-test resourceVersion: "18665" uid: cfc1ba67-3d13-4c9b-b80c-06182c8568ca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cc9bcc47d-zhtvs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:15:41Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n namespace: kserve-ci-e2e-test resourceVersion: "18672" uid: 60889480-945c-43a5-bf90-accfb84f6997 kind: Event lastTimestamp: "2026-07-20T10:15:41Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:15:41Z" name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n.18c3f7fca19d31ce namespace: kserve-ci-e2e-test resourceVersion: "18684" uid: 480f971a-accc-4703-8288-3d68f47e8ffd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:15:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n namespace: kserve-ci-e2e-test resourceVersion: "18657" uid: 60889480-945c-43a5-bf90-accfb84f6997 kind: Event lastTimestamp: "2026-07-20T10:15:41Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:038cd29f3e031cdfd5fa54ba896bbecc69cd255eda6b470c2f7bce5282357bb8" already present on machine metadata: creationTimestamp: "2026-07-20T10:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:15:41Z" name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n.18c3f7fca31baf90 namespace: kserve-ci-e2e-test resourceVersion: "18686" uid: 6abe67f2-cc31-46b4-8248-93fcc57b385f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:15:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n namespace: kserve-ci-e2e-test resourceVersion: "18657" uid: 60889480-945c-43a5-bf90-accfb84f6997 kind: Event lastTimestamp: "2026-07-20T10:15:41Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-20T10:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:15:41Z" name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n.18c3f7fca6778726 namespace: kserve-ci-e2e-test resourceVersion: "18687" uid: fed74ba1-82f7-48ef-ab05-365618195e25 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:15:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n namespace: kserve-ci-e2e-test resourceVersion: "18657" uid: 60889480-945c-43a5-bf90-accfb84f6997 kind: Event lastTimestamp: "2026-07-20T10:15:41Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-20T10:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:15:41Z" name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n.18c3f7fca70ee0b8 namespace: kserve-ci-e2e-test resourceVersion: "18688" uid: 90356039-ff9e-4bb5-9df8-9a0dd65758d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:15:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n namespace: kserve-ci-e2e-test resourceVersion: "18657" uid: 60889480-945c-43a5-bf90-accfb84f6997 kind: Event lastTimestamp: "2026-07-20T10:15:46Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:15:46Z" name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n.18c3f7fdc1589503 namespace: kserve-ci-e2e-test resourceVersion: "18727" uid: e8f6e900-1435-4830-b90a-ce13c197fa16 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:15:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n namespace: kserve-ci-e2e-test resourceVersion: "18657" uid: 60889480-945c-43a5-bf90-accfb84f6997 kind: Event lastTimestamp: "2026-07-20T10:15:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:15:46Z" name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n.18c3f7fdc5715a93 namespace: kserve-ci-e2e-test resourceVersion: "18729" uid: 8f9922ed-d29e-4123-b967-b156c2c1ef03 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:15:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n namespace: kserve-ci-e2e-test resourceVersion: "18657" uid: 60889480-945c-43a5-bf90-accfb84f6997 kind: Event lastTimestamp: "2026-07-20T10:15:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:15:46Z" name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n.18c3f7fdc5ea439f namespace: kserve-ci-e2e-test resourceVersion: "18732" uid: d6ab7315-9e5d-4f59-b07d-57fdfae01006 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:15:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n namespace: kserve-ci-e2e-test resourceVersion: "18657" uid: 60889480-945c-43a5-bf90-accfb84f6997 kind: Event lastTimestamp: "2026-07-20T10:15:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:15:46Z" name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n.18c3f7fdc5f57dc1 namespace: kserve-ci-e2e-test resourceVersion: "18733" uid: 2b1ee98c-1568-4df5-8f3b-97415ccfce86 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:15:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n namespace: kserve-ci-e2e-test resourceVersion: "18657" uid: 60889480-945c-43a5-bf90-accfb84f6997 kind: Event lastTimestamp: "2026-07-20T10:15:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:15:46Z" name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n.18c3f7fdc8d7151a namespace: kserve-ci-e2e-test resourceVersion: "18734" uid: c674f141-f0a4-4a2a-8a99-dfc7d9392415 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:15:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n namespace: kserve-ci-e2e-test resourceVersion: "18657" uid: 60889480-945c-43a5-bf90-accfb84f6997 kind: Event lastTimestamp: "2026-07-20T10:15:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:15:46Z" name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n.18c3f7fdc94fc9db namespace: kserve-ci-e2e-test resourceVersion: "18737" uid: 95d97676-fb58-457c-a3a7-41ef4a49b1bb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 20 eventTime: null firstTimestamp: "2026-07-20T10:15:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n namespace: kserve-ci-e2e-test resourceVersion: "18657" uid: 60889480-945c-43a5-bf90-accfb84f6997 kind: Event lastTimestamp: "2026-07-20T10:18:51Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:15:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:18:51Z" name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n.18c3f7fe3967cb06 namespace: kserve-ci-e2e-test resourceVersion: "20119" uid: 36740c41-b741-441b-b07f-f98c51de13f0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:18:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n namespace: kserve-ci-e2e-test resourceVersion: "18657" uid: 60889480-945c-43a5-bf90-accfb84f6997 kind: Event lastTimestamp: "2026-07-20T10:18:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:18:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:18:41Z" name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n.18c3f82675d4f538 namespace: kserve-ci-e2e-test resourceVersion: "20004" uid: 6b6d17af-ea93-48a5-905a-eaa9e6aecfe4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:18:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n namespace: kserve-ci-e2e-test resourceVersion: "18657" uid: 60889480-945c-43a5-bf90-accfb84f6997 kind: Event lastTimestamp: "2026-07-20T10:18:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:18:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:18:41Z" name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n.18c3f82675d574bf namespace: kserve-ci-e2e-test resourceVersion: "20005" uid: 3dfdb1e5-5d95-4a47-b183-113e8adaecc7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:18:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n namespace: kserve-ci-e2e-test resourceVersion: "18657" uid: 60889480-945c-43a5-bf90-accfb84f6997 kind: Event lastTimestamp: "2026-07-20T10:18:49Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:18:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:18:49Z" name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n.18c3f827341494a0 namespace: kserve-ci-e2e-test resourceVersion: "20104" uid: bc11b49e-8e4a-4a55-a641-2d1b347a6b39 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:15:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb namespace: kserve-ci-e2e-test resourceVersion: "18624" uid: 5826bd89-5655-473d-904d-26e45390739d kind: Event lastTimestamp: "2026-07-20T10:15:41Z" message: 'Created pod: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb-pp67n' metadata: creationTimestamp: "2026-07-20T10:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:15:41Z" name: isvc-sklearn-scale-raw-07030-predictor-888cc9fdb.18c3f7fc86a66664 namespace: kserve-ci-e2e-test resourceVersion: "18656" uid: 38648e33-107c-4c09-9491-5bb586e3ffaa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:15:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-07030-predictor namespace: kserve-ci-e2e-test resourceVersion: "18622" uid: 05d18119-47a9-4af6-8f60-86aa1bd6e4f5 kind: Event lastTimestamp: "2026-07-20T10:15:41Z" message: Scaled up replica set isvc-sklearn-scale-raw-07030-predictor-888cc9fdb from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:15:41Z" name: isvc-sklearn-scale-raw-07030-predictor.18c3f7fc808895c6 namespace: kserve-ci-e2e-test resourceVersion: "18626" uid: 314e83a5-2e02-4b96-a251-ca08d7422cee reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:15:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-07030-predictor namespace: kserve-ci-e2e-test resourceVersion: "18623" uid: 43eec5c2-4f54-426a-9312-90188f94e7f7 kind: Event lastTimestamp: "2026-07-20T10:16:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:15:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:16:26Z" name: isvc-sklearn-scale-raw-07030-predictor.18c3f7ffff99debb namespace: kserve-ci-e2e-test resourceVersion: "19039" uid: 83e8ac5f-fab2-4adb-bfe5-ad4d3537cf5a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:15:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-07030-predictor namespace: kserve-ci-e2e-test resourceVersion: "18623" uid: 43eec5c2-4f54-426a-9312-90188f94e7f7 kind: Event lastTimestamp: "2026-07-20T10:16:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:15:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:16:26Z" name: isvc-sklearn-scale-raw-07030-predictor.18c3f8000005af70 namespace: kserve-ci-e2e-test resourceVersion: "19040" uid: bbfe4fe0-032e-40b0-b523-e6e0e62e8f70 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-20T10:16:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-07030-predictor namespace: kserve-ci-e2e-test resourceVersion: "18816" uid: 43eec5c2-4f54-426a-9312-90188f94e7f7 kind: Event lastTimestamp: "2026-07-20T10:18:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:16:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:18:26Z" name: isvc-sklearn-scale-raw-07030-predictor.18c3f80a7d303a3a namespace: kserve-ci-e2e-test resourceVersion: "19872" uid: 1d38344b-bd7a-4daa-8505-290189f36524 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-20T10:16:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-07030-predictor namespace: kserve-ci-e2e-test resourceVersion: "18816" uid: 43eec5c2-4f54-426a-9312-90188f94e7f7 kind: Event lastTimestamp: "2026-07-20T10:18:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:16:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:18:26Z" name: isvc-sklearn-scale-raw-07030-predictor.18c3f80a7d9baa7c namespace: kserve-ci-e2e-test resourceVersion: "19873" uid: 790294a3-adc2-4c94-8ab5-1415ad6166d2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-20T10:15:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-07030 namespace: kserve-ci-e2e-test resourceVersion: "18615" uid: a157299e-b907-4822-9489-812b5658b00c kind: Event lastTimestamp: "2026-07-20T10:18:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:18:31Z" name: isvc-sklearn-scale-raw-07030.18c3f7fc80874d7f namespace: kserve-ci-e2e-test resourceVersion: "19919" uid: ea6c5140-5b2d-4c9a-8e51-44d60e4c6bdf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:18:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-07030 namespace: kserve-ci-e2e-test resourceVersion: "19916" uid: a157299e-b907-4822-9489-812b5658b00c kind: Event lastTimestamp: "2026-07-20T10:18:31Z" message: InferenceService [isvc-sklearn-scale-raw-07030] is Ready metadata: creationTimestamp: "2026-07-20T10:18:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:18:31Z" name: isvc-sklearn-scale-raw-07030.18c3f8242b21fdc5 namespace: kserve-ci-e2e-test resourceVersion: "19917" uid: a03368fd-4bec-4b6a-9839-702e187d80ec reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:10:05.171355Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb namespace: kserve-ci-e2e-test resourceVersion: "15608" uid: e734a3f6-4bdc-40f7-a417-8a5a72f7907f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb to ip-10-0-129-35.ec2.internal metadata: creationTimestamp: "2026-07-20T10:10:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:10:05Z" name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb.18c3f7ae41ee7b94 namespace: kserve-ci-e2e-test resourceVersion: "15614" uid: 6df8bd26-dd31-454f-9394-82503e862876 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cc9bcc47d-zhtvs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: e734a3f6-4bdc-40f7-a417-8a5a72f7907f kind: Event lastTimestamp: "2026-07-20T10:10:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-d73f8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:05Z" name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb.18c3f7ae4f4c5793 namespace: kserve-ci-e2e-test resourceVersion: "15629" uid: e0d8ee4a-d914-46ff-b775-a57b6020b9b5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: e734a3f6-4bdc-40f7-a417-8a5a72f7907f kind: Event lastTimestamp: "2026-07-20T10:10:06Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:10:06Z" name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb.18c3f7ae7fa0b850 namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: 05e76851-82f5-4f07-83d0-d4f6902d1b9a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: e734a3f6-4bdc-40f7-a417-8a5a72f7907f kind: Event lastTimestamp: "2026-07-20T10:10:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:038cd29f3e031cdfd5fa54ba896bbecc69cd255eda6b470c2f7bce5282357bb8" already present on machine metadata: creationTimestamp: "2026-07-20T10:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:06Z" name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb.18c3f7ae812544c0 namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: 0d78d5c9-9779-4f83-9dd7-ed4c4e8b0a91 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: e734a3f6-4bdc-40f7-a417-8a5a72f7907f kind: Event lastTimestamp: "2026-07-20T10:10:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-20T10:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:06Z" name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb.18c3f7ae84b802a0 namespace: kserve-ci-e2e-test resourceVersion: "15653" uid: cc407604-9dfb-4a6f-9ff0-3b1e7983ea30 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: e734a3f6-4bdc-40f7-a417-8a5a72f7907f kind: Event lastTimestamp: "2026-07-20T10:10:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-20T10:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:06Z" name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb.18c3f7ae8552610c namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: 992c2606-b083-43c9-ab57-2f5638e81761 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: e734a3f6-4bdc-40f7-a417-8a5a72f7907f kind: Event lastTimestamp: "2026-07-20T10:10:10Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-20T10:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:10Z" name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb.18c3f7af9caa1e62 namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: 638b18b6-a130-410b-8fff-dd8f50cc8ff8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: e734a3f6-4bdc-40f7-a417-8a5a72f7907f kind: Event lastTimestamp: "2026-07-20T10:10:28Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 17.796s (17.796s including waiting). Image size: 1306969339 bytes.' metadata: creationTimestamp: "2026-07-20T10:10:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:28Z" name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb.18c3f7b3c16c210e namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: 4cb7e810-caae-4f92-a8fc-20b68e4e9c04 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: e734a3f6-4bdc-40f7-a417-8a5a72f7907f kind: Event lastTimestamp: "2026-07-20T10:10:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:10:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:28Z" name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb.18c3f7b3c5a20c98 namespace: kserve-ci-e2e-test resourceVersion: "15853" uid: 0116c7dc-0049-4250-9140-cc86478d2363 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: e734a3f6-4bdc-40f7-a417-8a5a72f7907f kind: Event lastTimestamp: "2026-07-20T10:10:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:10:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:28Z" name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb.18c3f7b3c61c15fe namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: f2b902e4-6d6b-4ea6-bb72-efee344acb63 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: e734a3f6-4bdc-40f7-a417-8a5a72f7907f kind: Event lastTimestamp: "2026-07-20T10:10:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:10:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:28Z" name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb.18c3f7b3c627c3d4 namespace: kserve-ci-e2e-test resourceVersion: "15855" uid: 30592aca-93a5-4b05-9dfb-d53e7a184deb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: e734a3f6-4bdc-40f7-a417-8a5a72f7907f kind: Event lastTimestamp: "2026-07-20T10:10:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:10:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:28Z" name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb.18c3f7b3c9202d76 namespace: kserve-ci-e2e-test resourceVersion: "15856" uid: 1867170c-25ed-4e22-8a66-953dbe670718 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: e734a3f6-4bdc-40f7-a417-8a5a72f7907f kind: Event lastTimestamp: "2026-07-20T10:10:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:10:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:28Z" name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb.18c3f7b3c9957a2e namespace: kserve-ci-e2e-test resourceVersion: "15857" uid: 036e02bd-973a-4490-bef8-9887b1acf0c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-20T10:10:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: e734a3f6-4bdc-40f7-a417-8a5a72f7907f kind: Event lastTimestamp: "2026-07-20T10:11:56Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:10:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:56Z" name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb.18c3f7b40fdb82e4 namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: 9cf82587-ab48-486e-8f92-27b8b212b25a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: e734a3f6-4bdc-40f7-a417-8a5a72f7907f kind: Event lastTimestamp: "2026-07-20T10:11:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:55Z" name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb.18c3f7c7fe9a4bfd namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: d863a7c2-e031-4d0a-aa74-b512a9709464 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: e734a3f6-4bdc-40f7-a417-8a5a72f7907f kind: Event lastTimestamp: "2026-07-20T10:11:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:55Z" name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb.18c3f7c7fe9b64fe namespace: kserve-ci-e2e-test resourceVersion: "16671" uid: 1238b3a5-35fa-497c-89d2-ac81394e5f65 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: e734a3f6-4bdc-40f7-a417-8a5a72f7907f kind: Event lastTimestamp: "2026-07-20T10:11:56Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:56Z" name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb.18c3f7c81615177e namespace: kserve-ci-e2e-test resourceVersion: "16687" uid: 7d23de08-1e22-4945-9385-ab26633d9919 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c namespace: kserve-ci-e2e-test resourceVersion: "15595" uid: 9417c659-b454-4ee4-812a-d72449d73c44 kind: Event lastTimestamp: "2026-07-20T10:10:05Z" message: 'Created pod: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c-88kjb' metadata: creationTimestamp: "2026-07-20T10:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:05Z" name: isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c.18c3f7ae417aaeea namespace: kserve-ci-e2e-test resourceVersion: "15610" uid: 8ec2614e-e9ac-420e-b2c7-81359fdc4125 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-d73f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: cc9698b1-e332-404e-911d-da0e2a08b402 kind: Event lastTimestamp: "2026-07-20T10:10:05Z" message: Scaled up replica set isvc-xgboost-graph-raw-d73f8-predictor-5d6dc79f9c from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:05Z" name: isvc-xgboost-graph-raw-d73f8-predictor.18c3f7ae3e0053cf namespace: kserve-ci-e2e-test resourceVersion: "15597" uid: 424efae1-295d-4cc3-94ae-d865283cabf2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:10:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-d73f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15593" uid: ec8fe0b0-a630-48f1-a805-3b1de7d9d009 kind: Event lastTimestamp: "2026-07-20T10:11:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:05Z" name: isvc-xgboost-graph-raw-d73f8-predictor.18c3f7b1bce3bf68 namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: c569a79e-d408-4d57-bbcb-1ec6e011f0ba reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:10:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-d73f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15593" uid: ec8fe0b0-a630-48f1-a805-3b1de7d9d009 kind: Event lastTimestamp: "2026-07-20T10:11:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:05Z" name: isvc-xgboost-graph-raw-d73f8-predictor.18c3f7b1bd30f291 namespace: kserve-ci-e2e-test resourceVersion: "16125" uid: b64c71f2-2393-4d7a-822f-55e8ed6ca499 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:11:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-d73f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15787" uid: ec8fe0b0-a630-48f1-a805-3b1de7d9d009 kind: Event lastTimestamp: "2026-07-20T10:11:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:11:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:50Z" name: isvc-xgboost-graph-raw-d73f8-predictor.18c3f7bfb83027a5 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: ddae53e3-2782-413d-a9bf-eb8579f39a9c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:11:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-d73f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15787" uid: ec8fe0b0-a630-48f1-a805-3b1de7d9d009 kind: Event lastTimestamp: "2026-07-20T10:11:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:11:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:50Z" name: isvc-xgboost-graph-raw-d73f8-predictor.18c3f7bfb8a73525 namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: a003afab-f740-4154-aea0-6f2f553dc277 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-20T10:10:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-d73f8 namespace: kserve-ci-e2e-test resourceVersion: "15575" uid: 5231b3fb-edc8-4e94-aed2-a7c3eb53a186 kind: Event lastTimestamp: "2026-07-20T10:11:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:11:36Z" name: isvc-xgboost-graph-raw-d73f8.18c3f7ae3e87772f namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: 62fbf06d-1c26-44b6-b73e-5d6616fc6dcd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-d73f8 namespace: kserve-ci-e2e-test resourceVersion: "15575" uid: 5231b3fb-edc8-4e94-aed2-a7c3eb53a186 kind: Event lastTimestamp: "2026-07-20T10:10:05Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-d73f8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-d73f8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:10:05Z" name: isvc-xgboost-graph-raw-d73f8.18c3f7ae3f306580 namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 92cdc36b-4787-4bdb-9b15-906f60e07ceb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-d73f8 namespace: kserve-ci-e2e-test resourceVersion: "16375" uid: 5231b3fb-edc8-4e94-aed2-a7c3eb53a186 kind: Event lastTimestamp: "2026-07-20T10:11:36Z" message: InferenceService [isvc-xgboost-graph-raw-d73f8] is Ready metadata: creationTimestamp: "2026-07-20T10:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:11:36Z" name: isvc-xgboost-graph-raw-d73f8.18c3f7c3747e508d namespace: kserve-ci-e2e-test resourceVersion: "16377" uid: 7d030904-46de-483c-bab2-344cfe07c3f5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:11:55.574267Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb namespace: kserve-ci-e2e-test resourceVersion: "16633" uid: 352a995a-0109-4338-8ced-1bdef39b05e5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb to ip-10-0-128-168.ec2.internal metadata: creationTimestamp: "2026-07-20T10:11:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:11:55Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb.18c3f7c7f675251b namespace: kserve-ci-e2e-test resourceVersion: "16643" uid: a627d039-adee-4caf-9ad4-78a5700a10cb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cc9bcc47d-zhtvs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:55Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 352a995a-0109-4338-8ced-1bdef39b05e5 kind: Event lastTimestamp: "2026-07-20T10:11:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-hpa-0bd3d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:55Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb.18c3f7c800a297f9 namespace: kserve-ci-e2e-test resourceVersion: "16674" uid: 5babf476-cbb2-41f5-bf29-c31a48ff527b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-168.ec2.internal source: component: kubelet host: ip-10-0-128-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb namespace: kserve-ci-e2e-test resourceVersion: "16648" uid: 352a995a-0109-4338-8ced-1bdef39b05e5 kind: Event lastTimestamp: "2026-07-20T10:11:56Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:11:56Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb.18c3f7c833997afd namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: d7742ea6-a9d5-4354-8e98-734b590669d9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 352a995a-0109-4338-8ced-1bdef39b05e5 kind: Event lastTimestamp: "2026-07-20T10:11:56Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:038cd29f3e031cdfd5fa54ba896bbecc69cd255eda6b470c2f7bce5282357bb8" already present on machine metadata: creationTimestamp: "2026-07-20T10:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:56Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb.18c3f7c8358e10c8 namespace: kserve-ci-e2e-test resourceVersion: "16698" uid: fdedf359-f4d4-40a9-8e83-d80721ccde5c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-168.ec2.internal source: component: kubelet host: ip-10-0-128-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 352a995a-0109-4338-8ced-1bdef39b05e5 kind: Event lastTimestamp: "2026-07-20T10:11:56Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-20T10:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:56Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb.18c3f7c839455ca9 namespace: kserve-ci-e2e-test resourceVersion: "16700" uid: 2e692e29-7ca1-4dbf-8e80-2851d3107e91 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-168.ec2.internal source: component: kubelet host: ip-10-0-128-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 352a995a-0109-4338-8ced-1bdef39b05e5 kind: Event lastTimestamp: "2026-07-20T10:11:56Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-20T10:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:56Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb.18c3f7c839ab7aca namespace: kserve-ci-e2e-test resourceVersion: "16701" uid: 9be83c0f-3ddb-4afd-a544-c711dd014553 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-168.ec2.internal source: component: kubelet host: ip-10-0-128-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 352a995a-0109-4338-8ced-1bdef39b05e5 kind: Event lastTimestamp: "2026-07-20T10:12:00Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-20T10:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:12:00Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb.18c3f7c92ffd51e0 namespace: kserve-ci-e2e-test resourceVersion: "16752" uid: 059b1c93-3619-49d7-985f-44221dfa9ec3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-168.ec2.internal source: component: kubelet host: ip-10-0-128-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 352a995a-0109-4338-8ced-1bdef39b05e5 kind: Event lastTimestamp: "2026-07-20T10:12:19Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.069s (19.069s including waiting). Image size: 1306969339 bytes.' metadata: creationTimestamp: "2026-07-20T10:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:12:19Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb.18c3f7cda09a3623 namespace: kserve-ci-e2e-test resourceVersion: "16902" uid: e9082ccf-88d0-4a15-a060-88f16143e008 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-168.ec2.internal source: component: kubelet host: ip-10-0-128-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 352a995a-0109-4338-8ced-1bdef39b05e5 kind: Event lastTimestamp: "2026-07-20T10:12:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:12:19Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb.18c3f7cda3eb84a7 namespace: kserve-ci-e2e-test resourceVersion: "16904" uid: 7e0dd70c-cce9-431a-9719-8c1fa975092a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-168.ec2.internal source: component: kubelet host: ip-10-0-128-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 352a995a-0109-4338-8ced-1bdef39b05e5 kind: Event lastTimestamp: "2026-07-20T10:12:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:12:19Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb.18c3f7cda45cfc3b namespace: kserve-ci-e2e-test resourceVersion: "16905" uid: dfe50c3c-afde-4658-b5ad-c9e45287a972 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-168.ec2.internal source: component: kubelet host: ip-10-0-128-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 352a995a-0109-4338-8ced-1bdef39b05e5 kind: Event lastTimestamp: "2026-07-20T10:12:19Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-20T10:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:12:19Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb.18c3f7cda46a3644 namespace: kserve-ci-e2e-test resourceVersion: "16906" uid: e509473d-1697-4c5a-b622-cc373feed2e1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-168.ec2.internal source: component: kubelet host: ip-10-0-128-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 352a995a-0109-4338-8ced-1bdef39b05e5 kind: Event lastTimestamp: "2026-07-20T10:12:22Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.254s (2.254s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-20T10:12:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:12:22Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb.18c3f7ce2ac85f5b namespace: kserve-ci-e2e-test resourceVersion: "16924" uid: 8c994bfb-6236-484d-bcb4-083e0cac3f09 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-168.ec2.internal source: component: kubelet host: ip-10-0-128-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 352a995a-0109-4338-8ced-1bdef39b05e5 kind: Event lastTimestamp: "2026-07-20T10:12:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:12:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:12:22Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb.18c3f7ce2d9de520 namespace: kserve-ci-e2e-test resourceVersion: "16925" uid: 90a119c0-4047-4931-8bc6-994058318c58 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-168.ec2.internal source: component: kubelet host: ip-10-0-128-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 352a995a-0109-4338-8ced-1bdef39b05e5 kind: Event lastTimestamp: "2026-07-20T10:12:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:12:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:12:22Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb.18c3f7ce2e0c0fcb namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: fc4a5b9e-0939-44fb-b1b5-de8c892fab85 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-168.ec2.internal source: component: kubelet host: ip-10-0-128-168.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-20T10:12:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 352a995a-0109-4338-8ced-1bdef39b05e5 kind: Event lastTimestamp: "2026-07-20T10:13:09Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:12:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:13:09Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb.18c3f7ce9148fe49 namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 4ac7e283-8480-49d8-896e-c5a6715b0067 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-168.ec2.internal source: component: kubelet host: ip-10-0-128-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 352a995a-0109-4338-8ced-1bdef39b05e5 kind: Event lastTimestamp: "2026-07-20T10:13:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:13:46Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb.18c3f7e1c291e4c9 namespace: kserve-ci-e2e-test resourceVersion: "17681" uid: 388ae8bd-b66c-4271-8c0a-eaabeaa2030f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-168.ec2.internal source: component: kubelet host: ip-10-0-128-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 352a995a-0109-4338-8ced-1bdef39b05e5 kind: Event lastTimestamp: "2026-07-20T10:13:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:13:46Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb.18c3f7e1c2926de9 namespace: kserve-ci-e2e-test resourceVersion: "17682" uid: 657765e0-0529-425f-8a6d-18eecf032858 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-168.ec2.internal source: component: kubelet host: ip-10-0-128-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c namespace: kserve-ci-e2e-test resourceVersion: "16620" uid: 11e6b920-1ae5-43e4-957d-b290a9063e50 kind: Event lastTimestamp: "2026-07-20T10:11:55Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c-crbhb' metadata: creationTimestamp: "2026-07-20T10:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:55Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c.18c3f7c7f5bf69c4 namespace: kserve-ci-e2e-test resourceVersion: "16637" uid: f9fc1531-aea4-44fa-a275-409a637683a0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16615" uid: 229fd729-ea01-4211-becf-c900b8353041 kind: Event lastTimestamp: "2026-07-20T10:11:55Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-0bd3d-predictor-8556478b9c from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:55Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor.18c3f7c7f1c6e4c3 namespace: kserve-ci-e2e-test resourceVersion: "16624" uid: dc1fb8e2-19a9-46ab-8467-40bd7fc07f13 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:12:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: f40243c8-3262-4fb0-9817-bab0b9c392e5 kind: Event lastTimestamp: "2026-07-20T10:12:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:12:55Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor.18c3f7cb6fcf8765 namespace: kserve-ci-e2e-test resourceVersion: "17176" uid: 5e18250e-e0db-4691-a304-f98141cfd8e5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:12:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: f40243c8-3262-4fb0-9817-bab0b9c392e5 kind: Event lastTimestamp: "2026-07-20T10:12:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:12:55Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor.18c3f7cb70458783 namespace: kserve-ci-e2e-test resourceVersion: "17178" uid: 801902af-c78e-4e97-8137-db9ce02a0e96 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:13:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16841" uid: f40243c8-3262-4fb0-9817-bab0b9c392e5 kind: Event lastTimestamp: "2026-07-20T10:13:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:13:40Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor.18c3f7d96c73e187 namespace: kserve-ci-e2e-test resourceVersion: "17547" uid: 6c55df99-d6cc-4103-9134-ce76214f95ce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:13:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16841" uid: f40243c8-3262-4fb0-9817-bab0b9c392e5 kind: Event lastTimestamp: "2026-07-20T10:13:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:13:40Z" name: isvc-xgboost-graph-raw-hpa-0bd3d-predictor.18c3f7d96cee490e namespace: kserve-ci-e2e-test resourceVersion: "17548" uid: 0d9f1b0e-cb3d-4ab0-b2be-aca5b09833a3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-20T10:11:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-0bd3d namespace: kserve-ci-e2e-test resourceVersion: "16599" uid: 6dd532a7-9ebe-4b4c-b790-2715f12900f3 kind: Event lastTimestamp: "2026-07-20T10:13:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:13:20Z" name: isvc-xgboost-graph-raw-hpa-0bd3d.18c3f7c7f1a2eb35 namespace: kserve-ci-e2e-test resourceVersion: "17358" uid: 60d64778-cc89-410b-aaba-7dfc0a932647 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-0bd3d namespace: kserve-ci-e2e-test resourceVersion: "17356" uid: 6dd532a7-9ebe-4b4c-b790-2715f12900f3 kind: Event lastTimestamp: "2026-07-20T10:13:20Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-0bd3d] is Ready metadata: creationTimestamp: "2026-07-20T10:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:13:20Z" name: isvc-xgboost-graph-raw-hpa-0bd3d.18c3f7dba18a80b8 namespace: kserve-ci-e2e-test resourceVersion: "17357" uid: d5eafc03-4ac9-456c-b2c7-c0413312297a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:13:46.078430Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: a51afa2e-5ce2-42d9-99bc-30bda0468137 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-22c6a-predictor-7597674fb-vwxnt to ip-10-0-129-35.ec2.internal metadata: creationTimestamp: "2026-07-20T10:13:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:13:46Z" name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt.18c3f7e1b104e08e namespace: kserve-ci-e2e-test resourceVersion: "17637" uid: e3b687dd-7e67-4124-bdb5-22f143abdb01 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cc9bcc47d-zhtvs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:46Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt namespace: kserve-ci-e2e-test resourceVersion: "17645" uid: a51afa2e-5ce2-42d9-99bc-30bda0468137 kind: Event lastTimestamp: "2026-07-20T10:13:46Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:13:46Z" name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt.18c3f7e1cb2f0a1a namespace: kserve-ci-e2e-test resourceVersion: "17685" uid: 8f21e949-7b41-45a2-a009-4de7581c8886 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt namespace: kserve-ci-e2e-test resourceVersion: "17634" uid: a51afa2e-5ce2-42d9-99bc-30bda0468137 kind: Event lastTimestamp: "2026-07-20T10:13:46Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-20T10:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:13:46Z" name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt.18c3f7e1cd28f106 namespace: kserve-ci-e2e-test resourceVersion: "17687" uid: 9d07bd11-53c9-4d43-9dd3-c2d3ebf1cae9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt namespace: kserve-ci-e2e-test resourceVersion: "17634" uid: a51afa2e-5ce2-42d9-99bc-30bda0468137 kind: Event lastTimestamp: "2026-07-20T10:13:47Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.279s (1.279s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-20T10:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:13:47Z" name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt.18c3f7e2196cdb9c namespace: kserve-ci-e2e-test resourceVersion: "17696" uid: 2478a9d5-885f-42e0-98cc-9ebbaba23af2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt namespace: kserve-ci-e2e-test resourceVersion: "17634" uid: a51afa2e-5ce2-42d9-99bc-30bda0468137 kind: Event lastTimestamp: "2026-07-20T10:13:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:13:47Z" name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt.18c3f7e21c857dbd namespace: kserve-ci-e2e-test resourceVersion: "17697" uid: 9b1f33fd-3d1c-4a14-88ba-f5306108e722 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt namespace: kserve-ci-e2e-test resourceVersion: "17634" uid: a51afa2e-5ce2-42d9-99bc-30bda0468137 kind: Event lastTimestamp: "2026-07-20T10:13:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:13:47Z" name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt.18c3f7e21cfdd0ea namespace: kserve-ci-e2e-test resourceVersion: "17698" uid: 25643e2d-f9df-4155-8cff-69729e5b4ad2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt namespace: kserve-ci-e2e-test resourceVersion: "17634" uid: a51afa2e-5ce2-42d9-99bc-30bda0468137 kind: Event lastTimestamp: "2026-07-20T10:13:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:13:47Z" name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt.18c3f7e21d07bec7 namespace: kserve-ci-e2e-test resourceVersion: "17699" uid: 0f86986c-d3bd-42c9-b93c-ecfe100a3070 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt namespace: kserve-ci-e2e-test resourceVersion: "17634" uid: a51afa2e-5ce2-42d9-99bc-30bda0468137 kind: Event lastTimestamp: "2026-07-20T10:13:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:13:47Z" name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt.18c3f7e220f5ec54 namespace: kserve-ci-e2e-test resourceVersion: "17701" uid: 3484284b-3ec5-4962-b4c6-3de7f3529c45 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt namespace: kserve-ci-e2e-test resourceVersion: "17634" uid: a51afa2e-5ce2-42d9-99bc-30bda0468137 kind: Event lastTimestamp: "2026-07-20T10:13:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:13:47Z" name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt.18c3f7e22171397f namespace: kserve-ci-e2e-test resourceVersion: "17703" uid: 7d1a1d55-df92-4ef0-9603-d4cb9066987a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:15:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt namespace: kserve-ci-e2e-test resourceVersion: "17634" uid: a51afa2e-5ce2-42d9-99bc-30bda0468137 kind: Event lastTimestamp: "2026-07-20T10:15:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:15:41Z" name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt.18c3f7fc89bb9390 namespace: kserve-ci-e2e-test resourceVersion: "18674" uid: f0e49893-784f-4af9-9b0b-8a6ad308d208 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:15:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt namespace: kserve-ci-e2e-test resourceVersion: "17634" uid: a51afa2e-5ce2-42d9-99bc-30bda0468137 kind: Event lastTimestamp: "2026-07-20T10:15:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:15:41Z" name: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt.18c3f7fc89bcdf3a namespace: kserve-ci-e2e-test resourceVersion: "18678" uid: 402e6ac1-7b2e-43bc-aaa7-679eba3811da reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-22c6a-predictor-7597674fb namespace: kserve-ci-e2e-test resourceVersion: "17618" uid: 4721fe06-67a5-46c8-8b56-2eabbdd2aee4 kind: Event lastTimestamp: "2026-07-20T10:13:46Z" message: 'Created pod: message-dumper-raw-22c6a-predictor-7597674fb-vwxnt' metadata: creationTimestamp: "2026-07-20T10:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:13:46Z" name: message-dumper-raw-22c6a-predictor-7597674fb.18c3f7e1afd0c6a9 namespace: kserve-ci-e2e-test resourceVersion: "17635" uid: a00e33ea-b513-4874-b98d-42d4d70acc17 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-22c6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17616" uid: d058de9d-f82b-4c34-b791-6b4c82b36261 kind: Event lastTimestamp: "2026-07-20T10:13:45Z" message: Scaled up replica set message-dumper-raw-22c6a-predictor-7597674fb from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:13:45Z" name: message-dumper-raw-22c6a-predictor.18c3f7e1a60cdbb9 namespace: kserve-ci-e2e-test resourceVersion: "17623" uid: badfa1ea-6fc1-4f33-ab50-c25f7bea54de reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:14:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-22c6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17617" uid: a5b3da41-6299-4c2e-af31-7a76f38c90cd kind: Event lastTimestamp: "2026-07-20T10:14:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:14:15Z" name: message-dumper-raw-22c6a-predictor.18c3f7e5253da36e namespace: kserve-ci-e2e-test resourceVersion: "17992" uid: 5e91fe65-50ab-4804-b3a0-b40ee826dfdf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:14:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-22c6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17617" uid: a5b3da41-6299-4c2e-af31-7a76f38c90cd kind: Event lastTimestamp: "2026-07-20T10:14:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:14:15Z" name: message-dumper-raw-22c6a-predictor.18c3f7e525b52452 namespace: kserve-ci-e2e-test resourceVersion: "17994" uid: a40d1963-6794-4c13-89c4-68af0136d703 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:14:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-22c6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17812" uid: a5b3da41-6299-4c2e-af31-7a76f38c90cd kind: Event lastTimestamp: "2026-07-20T10:14:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:14:30Z" name: message-dumper-raw-22c6a-predictor.18c3f7ec24496fa5 namespace: kserve-ci-e2e-test resourceVersion: "18100" uid: 28ef1934-b844-4988-8bb1-53f7c2ad9643 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:14:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-22c6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17812" uid: a5b3da41-6299-4c2e-af31-7a76f38c90cd kind: Event lastTimestamp: "2026-07-20T10:14:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:14:30Z" name: message-dumper-raw-22c6a-predictor.18c3f7ec24bbd65f namespace: kserve-ci-e2e-test resourceVersion: "18102" uid: ce4893a8-8d0c-4e93-b593-c25892be79a6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-20T10:13:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-22c6a namespace: kserve-ci-e2e-test resourceVersion: "17610" uid: 68394d7d-7a87-4e9a-a75e-de75462ba199 kind: Event lastTimestamp: "2026-07-20T10:13:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:13:56Z" name: message-dumper-raw-22c6a.18c3f7e1a68cc563 namespace: kserve-ci-e2e-test resourceVersion: "17780" uid: fbc5fde2-9fb6-423c-a8f4-50ade429fc40 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-22c6a namespace: kserve-ci-e2e-test resourceVersion: "17610" uid: 68394d7d-7a87-4e9a-a75e-de75462ba199 kind: Event lastTimestamp: "2026-07-20T10:13:45Z" message: 'Failed to update status for InferenceService "message-dumper-raw-22c6a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-22c6a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:13:45Z" name: message-dumper-raw-22c6a.18c3f7e1a8196dc1 namespace: kserve-ci-e2e-test resourceVersion: "17624" uid: 21179951-00e5-4ae3-b1e4-c03f148c2161 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-22c6a namespace: kserve-ci-e2e-test resourceVersion: "17778" uid: 68394d7d-7a87-4e9a-a75e-de75462ba199 kind: Event lastTimestamp: "2026-07-20T10:13:56Z" message: InferenceService [message-dumper-raw-22c6a] is Ready metadata: creationTimestamp: "2026-07-20T10:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:13:56Z" name: message-dumper-raw-22c6a.18c3f7e4098dc1c6 namespace: kserve-ci-e2e-test resourceVersion: "17779" uid: 33927215-2126-411e-af94-98689c5bee33 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:11:45.115214Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-d73f8-56cffcf457-9bc2n namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: d9dc2fa6-3092-45ea-ba0b-880c4df26f18 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-d73f8-56cffcf457-9bc2n to ip-10-0-142-247.ec2.internal metadata: creationTimestamp: "2026-07-20T10:11:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:11:45Z" name: model-chainer-raw-d73f8-56cffcf457-9bc2n.18c3f7c5870ca7e3 namespace: kserve-ci-e2e-test resourceVersion: "16455" uid: bdebae23-f6aa-4e59-ac56-25c2b229aa71 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cc9bcc47d-zhtvs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:45Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-d73f8-56cffcf457-9bc2n namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: d9dc2fa6-3092-45ea-ba0b-880c4df26f18 kind: Event lastTimestamp: "2026-07-20T10:11:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-d73f8-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:45Z" name: model-chainer-raw-d73f8-56cffcf457-9bc2n.18c3f7c590120a10 namespace: kserve-ci-e2e-test resourceVersion: "16466" uid: a7a8c756-cbf9-4f01-b2e1-f5020dccbc00 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-247.ec2.internal source: component: kubelet host: ip-10-0-142-247.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:46Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-d73f8-56cffcf457-9bc2n namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: d9dc2fa6-3092-45ea-ba0b-880c4df26f18 kind: Event lastTimestamp: "2026-07-20T10:11:46Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:11:46Z" name: model-chainer-raw-d73f8-56cffcf457-9bc2n.18c3f7c5c52f0d7f namespace: kserve-ci-e2e-test resourceVersion: "16476" uid: 5698153f-6fa6-461d-b703-212c6b5d7c54 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-d73f8} kind: Pod name: model-chainer-raw-d73f8-56cffcf457-9bc2n namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: d9dc2fa6-3092-45ea-ba0b-880c4df26f18 kind: Event lastTimestamp: "2026-07-20T10:11:46Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:ee73d7a22df8b79aa8a5e917cb435b1d2b100d4582439c437bdc01a1d2422290" metadata: creationTimestamp: "2026-07-20T10:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:46Z" name: model-chainer-raw-d73f8-56cffcf457-9bc2n.18c3f7c5c6846789 namespace: kserve-ci-e2e-test resourceVersion: "16478" uid: f206e52b-cac3-4d9f-bc25-78105baee196 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-247.ec2.internal source: component: kubelet host: ip-10-0-142-247.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-d73f8} kind: Pod name: model-chainer-raw-d73f8-56cffcf457-9bc2n namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: d9dc2fa6-3092-45ea-ba0b-880c4df26f18 kind: Event lastTimestamp: "2026-07-20T10:11:48Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:ee73d7a22df8b79aa8a5e917cb435b1d2b100d4582439c437bdc01a1d2422290" in 2.12s (2.12s including waiting). Image size: 218228855 bytes.' metadata: creationTimestamp: "2026-07-20T10:11:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:48Z" name: model-chainer-raw-d73f8-56cffcf457-9bc2n.18c3f7c644e72f00 namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: c577c2a5-e71a-4232-aecf-385f915bde43 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-247.ec2.internal source: component: kubelet host: ip-10-0-142-247.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-d73f8} kind: Pod name: model-chainer-raw-d73f8-56cffcf457-9bc2n namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: d9dc2fa6-3092-45ea-ba0b-880c4df26f18 kind: Event lastTimestamp: "2026-07-20T10:11:48Z" message: 'Created container: model-chainer-raw-d73f8' metadata: creationTimestamp: "2026-07-20T10:11:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:48Z" name: model-chainer-raw-d73f8-56cffcf457-9bc2n.18c3f7c647bcbb80 namespace: kserve-ci-e2e-test resourceVersion: "16493" uid: 2344ace5-4188-4581-aa2d-7571789b12e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-247.ec2.internal source: component: kubelet host: ip-10-0-142-247.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-d73f8} kind: Pod name: model-chainer-raw-d73f8-56cffcf457-9bc2n namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: d9dc2fa6-3092-45ea-ba0b-880c4df26f18 kind: Event lastTimestamp: "2026-07-20T10:11:48Z" message: Started container model-chainer-raw-d73f8 metadata: creationTimestamp: "2026-07-20T10:11:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:48Z" name: model-chainer-raw-d73f8-56cffcf457-9bc2n.18c3f7c64828cead namespace: kserve-ci-e2e-test resourceVersion: "16495" uid: ea4a9701-9901-436a-95bb-2dd827dbae3d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-247.ec2.internal source: component: kubelet host: ip-10-0-142-247.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-d73f8} kind: Pod name: model-chainer-raw-d73f8-56cffcf457-9bc2n namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: d9dc2fa6-3092-45ea-ba0b-880c4df26f18 kind: Event lastTimestamp: "2026-07-20T10:11:55Z" message: Stopping container model-chainer-raw-d73f8 metadata: creationTimestamp: "2026-07-20T10:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:55Z" name: model-chainer-raw-d73f8-56cffcf457-9bc2n.18c3f7c7defb8fc3 namespace: kserve-ci-e2e-test resourceVersion: "16563" uid: 41db099e-e599-4a16-9c16-0cfb281ae4dd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-247.ec2.internal source: component: kubelet host: ip-10-0-142-247.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:11:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-d73f8} kind: Pod name: model-chainer-raw-d73f8-56cffcf457-9bc2n namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: d9dc2fa6-3092-45ea-ba0b-880c4df26f18 kind: Event lastTimestamp: "2026-07-20T10:12:24Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-20T10:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:12:24Z" name: model-chainer-raw-d73f8-56cffcf457-9bc2n.18c3f7c8ea0a5369 namespace: kserve-ci-e2e-test resourceVersion: "16945" uid: 280bf630-fe0d-4a6a-a4e5-bb1571e90f7f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-247.ec2.internal source: component: kubelet host: ip-10-0-142-247.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-d73f8-56cffcf457 namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: f5042b63-a4dd-488a-8392-295578e25033 kind: Event lastTimestamp: "2026-07-20T10:11:45Z" message: 'Created pod: model-chainer-raw-d73f8-56cffcf457-9bc2n' metadata: creationTimestamp: "2026-07-20T10:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:45Z" name: model-chainer-raw-d73f8-56cffcf457.18c3f7c58684ab51 namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: be7c00c7-d071-44f3-bdcf-744ff35c9037 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-d73f8 namespace: kserve-ci-e2e-test resourceVersion: "16444" uid: c745d852-b901-4432-a398-fdcd8a011623 kind: Event lastTimestamp: "2026-07-20T10:11:45Z" message: Scaled up replica set model-chainer-raw-d73f8-56cffcf457 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:45Z" name: model-chainer-raw-d73f8.18c3f7c5857f22e7 namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: e1b1c9da-b165-4e49-84f3-646919fcc22d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-d73f8 namespace: kserve-ci-e2e-test resourceVersion: "16461" uid: ced62da1-9c70-47e4-89dc-38810380f993 kind: Event lastTimestamp: "2026-07-20T10:11:45Z" message: InferenceGraph [model-chainer-raw-d73f8] is Ready metadata: creationTimestamp: "2026-07-20T10:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:11:45Z" name: model-chainer-raw-d73f8.18c3f7c58bf5b881 namespace: kserve-ci-e2e-test resourceVersion: "16463" uid: 045fb269-d817-4442-b5ea-feffe9ba61ea reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:13:35.502607Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-0bd3d-87697b988-dn4jm namespace: kserve-ci-e2e-test resourceVersion: "17484" uid: 479b2dad-c660-4e2f-8417-4662966f580c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-0bd3d-87697b988-dn4jm to ip-10-0-142-247.ec2.internal metadata: creationTimestamp: "2026-07-20T10:13:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:13:35Z" name: model-chainer-raw-hpa-0bd3d-87697b988-dn4jm.18c3f7df3aa696cd namespace: kserve-ci-e2e-test resourceVersion: "17489" uid: 40658c3e-bb73-4775-a205-2429e903b4c8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cc9bcc47d-zhtvs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:35Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-0bd3d-87697b988-dn4jm namespace: kserve-ci-e2e-test resourceVersion: "17488" uid: 479b2dad-c660-4e2f-8417-4662966f580c kind: Event lastTimestamp: "2026-07-20T10:13:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-0bd3d-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:13:35Z" name: model-chainer-raw-hpa-0bd3d-87697b988-dn4jm.18c3f7df46d0979a namespace: kserve-ci-e2e-test resourceVersion: "17500" uid: 1ffeee15-162a-44a7-8f49-f42f376d3f18 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-247.ec2.internal source: component: kubelet host: ip-10-0-142-247.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:36Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-0bd3d-87697b988-dn4jm namespace: kserve-ci-e2e-test resourceVersion: "17492" uid: 479b2dad-c660-4e2f-8417-4662966f580c kind: Event lastTimestamp: "2026-07-20T10:13:36Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:13:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:13:36Z" name: model-chainer-raw-hpa-0bd3d-87697b988-dn4jm.18c3f7df77adfb9f namespace: kserve-ci-e2e-test resourceVersion: "17510" uid: 59dcc305-1bfb-4ac1-9b19-fcdf264f8135 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-0bd3d} kind: Pod name: model-chainer-raw-hpa-0bd3d-87697b988-dn4jm namespace: kserve-ci-e2e-test resourceVersion: "17488" uid: 479b2dad-c660-4e2f-8417-4662966f580c kind: Event lastTimestamp: "2026-07-20T10:13:36Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ee73d7a22df8b79aa8a5e917cb435b1d2b100d4582439c437bdc01a1d2422290" already present on machine metadata: creationTimestamp: "2026-07-20T10:13:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:13:36Z" name: model-chainer-raw-hpa-0bd3d-87697b988-dn4jm.18c3f7df790e2e14 namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: c2a8b7a5-5b78-4867-b203-4605d1e7f8a8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-247.ec2.internal source: component: kubelet host: ip-10-0-142-247.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-0bd3d} kind: Pod name: model-chainer-raw-hpa-0bd3d-87697b988-dn4jm namespace: kserve-ci-e2e-test resourceVersion: "17488" uid: 479b2dad-c660-4e2f-8417-4662966f580c kind: Event lastTimestamp: "2026-07-20T10:13:36Z" message: 'Created container: model-chainer-raw-hpa-0bd3d' metadata: creationTimestamp: "2026-07-20T10:13:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:13:36Z" name: model-chainer-raw-hpa-0bd3d-87697b988-dn4jm.18c3f7df7bd82f24 namespace: kserve-ci-e2e-test resourceVersion: "17513" uid: 80403c62-9981-4569-ae65-81f467f8c63f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-247.ec2.internal source: component: kubelet host: ip-10-0-142-247.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-0bd3d} kind: Pod name: model-chainer-raw-hpa-0bd3d-87697b988-dn4jm namespace: kserve-ci-e2e-test resourceVersion: "17488" uid: 479b2dad-c660-4e2f-8417-4662966f580c kind: Event lastTimestamp: "2026-07-20T10:13:36Z" message: Started container model-chainer-raw-hpa-0bd3d metadata: creationTimestamp: "2026-07-20T10:13:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:13:36Z" name: model-chainer-raw-hpa-0bd3d-87697b988-dn4jm.18c3f7df7c44ce82 namespace: kserve-ci-e2e-test resourceVersion: "17514" uid: caf8f299-5aac-46bc-85c8-65bb9a13a533 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-247.ec2.internal source: component: kubelet host: ip-10-0-142-247.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-0bd3d} kind: Pod name: model-chainer-raw-hpa-0bd3d-87697b988-dn4jm namespace: kserve-ci-e2e-test resourceVersion: "17488" uid: 479b2dad-c660-4e2f-8417-4662966f580c kind: Event lastTimestamp: "2026-07-20T10:13:45Z" message: Stopping container model-chainer-raw-hpa-0bd3d metadata: creationTimestamp: "2026-07-20T10:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:13:45Z" name: model-chainer-raw-hpa-0bd3d-87697b988-dn4jm.18c3f7e1953cc631 namespace: kserve-ci-e2e-test resourceVersion: "17605" uid: 09e9a2bb-c2df-4e09-9360-d1a741659fe9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-247.ec2.internal source: component: kubelet host: ip-10-0-142-247.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:13:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-0bd3d} kind: Pod name: model-chainer-raw-hpa-0bd3d-87697b988-dn4jm namespace: kserve-ci-e2e-test resourceVersion: "17488" uid: 479b2dad-c660-4e2f-8417-4662966f580c kind: Event lastTimestamp: "2026-07-20T10:14:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-20T10:13:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:14:13Z" name: model-chainer-raw-hpa-0bd3d-87697b988-dn4jm.18c3f7e225012f90 namespace: kserve-ci-e2e-test resourceVersion: "17967" uid: 061e0934-6d04-48ce-904e-afb48539b57b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-247.ec2.internal source: component: kubelet host: ip-10-0-142-247.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-0bd3d-87697b988 namespace: kserve-ci-e2e-test resourceVersion: "17481" uid: e016fc50-6097-4794-b863-873278b55e64 kind: Event lastTimestamp: "2026-07-20T10:13:35Z" message: 'Created pod: model-chainer-raw-hpa-0bd3d-87697b988-dn4jm' metadata: creationTimestamp: "2026-07-20T10:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:13:35Z" name: model-chainer-raw-hpa-0bd3d-87697b988.18c3f7df39e448f2 namespace: kserve-ci-e2e-test resourceVersion: "17486" uid: 61b34a9c-5557-429a-a4c4-cacccbe4beba reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-0bd3d namespace: kserve-ci-e2e-test resourceVersion: "17479" uid: 69f604a1-f039-4ae3-b7ef-8f880fd226ec kind: Event lastTimestamp: "2026-07-20T10:13:35Z" message: Scaled up replica set model-chainer-raw-hpa-0bd3d-87697b988 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:13:35Z" name: model-chainer-raw-hpa-0bd3d.18c3f7df38018915 namespace: kserve-ci-e2e-test resourceVersion: "17482" uid: 2570f5b8-b2d5-48df-bf38-c3711ed1953d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:13:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-0bd3d namespace: kserve-ci-e2e-test resourceVersion: "17495" uid: 36569316-e6b0-46e6-8475-d96b6bd89101 kind: Event lastTimestamp: "2026-07-20T10:13:35Z" message: InferenceGraph [model-chainer-raw-hpa-0bd3d] is Ready metadata: creationTimestamp: "2026-07-20T10:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:13:35Z" name: model-chainer-raw-hpa-0bd3d.18c3f7df3e0ee79d namespace: kserve-ci-e2e-test resourceVersion: "17497" uid: 636a5211-4861-446f-9a32-cc823ce8af44 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:21:10.474040Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx namespace: kserve-ci-e2e-test resourceVersion: "21332" uid: c1b22526-ac03-4063-8fc3-10aeba6ba4b2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx to ip-10-0-129-35.ec2.internal metadata: creationTimestamp: "2026-07-20T10:21:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:21:10Z" name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx.18c3f849290fc5f9 namespace: kserve-ci-e2e-test resourceVersion: "21338" uid: 0a052a6f-b0e5-4482-8aa1-52e8736daed4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cc9bcc47d-zhtvs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:10Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx namespace: kserve-ci-e2e-test resourceVersion: "21342" uid: c1b22526-ac03-4063-8fc3-10aeba6ba4b2 kind: Event lastTimestamp: "2026-07-20T10:21:10Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:21:10Z" name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx.18c3f84943fa458a namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: e2569115-ee19-4d6b-af4e-ea25cf1588a2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: c1b22526-ac03-4063-8fc3-10aeba6ba4b2 kind: Event lastTimestamp: "2026-07-20T10:21:10Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:038cd29f3e031cdfd5fa54ba896bbecc69cd255eda6b470c2f7bce5282357bb8" already present on machine metadata: creationTimestamp: "2026-07-20T10:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:10Z" name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx.18c3f84945f0faa1 namespace: kserve-ci-e2e-test resourceVersion: "21354" uid: 514b8cab-ef66-41a7-a3e2-6ebd0fac848e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: c1b22526-ac03-4063-8fc3-10aeba6ba4b2 kind: Event lastTimestamp: "2026-07-20T10:21:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-20T10:21:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:11Z" name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx.18c3f84949e07d24 namespace: kserve-ci-e2e-test resourceVersion: "21356" uid: 28fa9ab9-f52a-4e20-9855-9cd269751f94 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: c1b22526-ac03-4063-8fc3-10aeba6ba4b2 kind: Event lastTimestamp: "2026-07-20T10:21:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-20T10:21:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:11Z" name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx.18c3f8494a5c7a2c namespace: kserve-ci-e2e-test resourceVersion: "21357" uid: 8e72d80d-ce29-4706-abe0-60dce99eb512 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: c1b22526-ac03-4063-8fc3-10aeba6ba4b2 kind: Event lastTimestamp: "2026-07-20T10:21:15Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:15Z" name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx.18c3f84a3fee0dd6 namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: 09cc5eb2-e29a-40cd-88cc-351f75f62e52 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: c1b22526-ac03-4063-8fc3-10aeba6ba4b2 kind: Event lastTimestamp: "2026-07-20T10:21:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:15Z" name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx.18c3f84a4328c44a namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: 68fc558f-8500-4937-8dda-964b07a12d95 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: c1b22526-ac03-4063-8fc3-10aeba6ba4b2 kind: Event lastTimestamp: "2026-07-20T10:21:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:15Z" name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx.18c3f84a43a31854 namespace: kserve-ci-e2e-test resourceVersion: "21394" uid: 4402c4bf-eb40-4587-9b1d-a5a40105623d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: c1b22526-ac03-4063-8fc3-10aeba6ba4b2 kind: Event lastTimestamp: "2026-07-20T10:21:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:15Z" name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx.18c3f84a43ae6579 namespace: kserve-ci-e2e-test resourceVersion: "21395" uid: 6ee6521a-d3c1-4294-91ea-1999595d9e02 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: c1b22526-ac03-4063-8fc3-10aeba6ba4b2 kind: Event lastTimestamp: "2026-07-20T10:21:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:15Z" name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx.18c3f84a4776eb83 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: e1953010-cbaa-430f-be71-ea16b3b3993d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: c1b22526-ac03-4063-8fc3-10aeba6ba4b2 kind: Event lastTimestamp: "2026-07-20T10:21:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:15Z" name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx.18c3f84a47ed8ed1 namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: b462b161-5aba-43e1-943b-351de622f327 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-20T10:21:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: c1b22526-ac03-4063-8fc3-10aeba6ba4b2 kind: Event lastTimestamp: "2026-07-20T10:22:42Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:21:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:22:42Z" name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx.18c3f84a7bf95592 namespace: kserve-ci-e2e-test resourceVersion: "22037" uid: 9e0e6c4b-66bf-48ad-b029-07a7a3248e79 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: c1b22526-ac03-4063-8fc3-10aeba6ba4b2 kind: Event lastTimestamp: "2026-07-20T10:22:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:22:40Z" name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx.18c3f85e245e62f7 namespace: kserve-ci-e2e-test resourceVersion: "22024" uid: b1eac8d4-81bc-4fde-ba68-d9dd23537bf4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: c1b22526-ac03-4063-8fc3-10aeba6ba4b2 kind: Event lastTimestamp: "2026-07-20T10:22:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:22:40Z" name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx.18c3f85e245f3d72 namespace: kserve-ci-e2e-test resourceVersion: "22025" uid: 29ca33ff-d11b-4ed0-8011-cac19f356030 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: c1b22526-ac03-4063-8fc3-10aeba6ba4b2 kind: Event lastTimestamp: "2026-07-20T10:22:42Z" message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": dial tcp 10.133.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:22:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:22:42Z" name: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx.18c3f85e8227168b namespace: kserve-ci-e2e-test resourceVersion: "22036" uid: 3fbf9772-a32f-44b4-87f6-ae2ab92f6bc4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-2ee4d-predictor-67d5f8f496 namespace: kserve-ci-e2e-test resourceVersion: "21323" uid: 88dc591d-ed66-4d17-adff-0547989eb839 kind: Event lastTimestamp: "2026-07-20T10:21:10Z" message: 'Created pod: raw-sklearn-2ee4d-predictor-67d5f8f496-wbcqx' metadata: creationTimestamp: "2026-07-20T10:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:10Z" name: raw-sklearn-2ee4d-predictor-67d5f8f496.18c3f84928786f94 namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: 9f748874-46db-4ffb-b59f-628299b71a77 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-2ee4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21321" uid: 73417046-202f-43ed-be19-7ad5e507f42a kind: Event lastTimestamp: "2026-07-20T10:21:10Z" message: Scaled up replica set raw-sklearn-2ee4d-predictor-67d5f8f496 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:10Z" name: raw-sklearn-2ee4d-predictor.18c3f8492559a199 namespace: kserve-ci-e2e-test resourceVersion: "21325" uid: fc37fdc4-80ca-44af-b36c-37eb8e654144 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-2ee4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21322" uid: c80a8af7-57c2-476c-9402-e8f0fe806fa4 kind: Event lastTimestamp: "2026-07-20T10:21:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:25Z" name: raw-sklearn-2ee4d-predictor.18c3f84ca3d5d2ce namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: 7ab35c00-d5da-4b17-9b93-15be46d8db11 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-2ee4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21322" uid: c80a8af7-57c2-476c-9402-e8f0fe806fa4 kind: Event lastTimestamp: "2026-07-20T10:21:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:25Z" name: raw-sklearn-2ee4d-predictor.18c3f84ca447b5b1 namespace: kserve-ci-e2e-test resourceVersion: "21470" uid: 1770a019-2d44-4bb1-af9c-fe557184cdea reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:21:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-2ee4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21469" uid: c80a8af7-57c2-476c-9402-e8f0fe806fa4 kind: Event lastTimestamp: "2026-07-20T10:22:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:22:40Z" name: raw-sklearn-2ee4d-predictor.18c3f850238bb9d8 namespace: kserve-ci-e2e-test resourceVersion: "22008" uid: 44a05eff-09bf-4d3f-9209-d1d663d4ace0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:21:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-2ee4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21469" uid: c80a8af7-57c2-476c-9402-e8f0fe806fa4 kind: Event lastTimestamp: "2026-07-20T10:22:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:22:40Z" name: raw-sklearn-2ee4d-predictor.18c3f85023fe6fc0 namespace: kserve-ci-e2e-test resourceVersion: "22009" uid: 66ec65e1-6b6b-483a-bec2-bd80a95e4a7f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-20T10:21:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-2ee4d namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: 5b14f361-a294-49d5-926d-72bee07bc99e kind: Event lastTimestamp: "2026-07-20T10:22:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:22:32Z" name: raw-sklearn-2ee4d.18c3f8492559a57c namespace: kserve-ci-e2e-test resourceVersion: "21951" uid: 324a4ec6-b0b4-4b46-8232-8a3ed7c6fa63 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-2ee4d namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: 5b14f361-a294-49d5-926d-72bee07bc99e kind: Event lastTimestamp: "2026-07-20T10:21:10Z" message: 'Failed to update status for InferenceService "raw-sklearn-2ee4d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-2ee4d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:21:10Z" name: raw-sklearn-2ee4d.18c3f84925dd58ca namespace: kserve-ci-e2e-test resourceVersion: "21327" uid: 9d165b3a-61c0-40e4-94e4-082d4c103048 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-2ee4d namespace: kserve-ci-e2e-test resourceVersion: "21949" uid: 5b14f361-a294-49d5-926d-72bee07bc99e kind: Event lastTimestamp: "2026-07-20T10:22:32Z" message: InferenceService [raw-sklearn-2ee4d] is Ready metadata: creationTimestamp: "2026-07-20T10:22:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:22:32Z" name: raw-sklearn-2ee4d.18c3f85c33575b77 namespace: kserve-ci-e2e-test resourceVersion: "21950" uid: fd935522-b8ac-44d6-8fc6-99fc1d8ee789 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:22:45.765942Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq namespace: kserve-ci-e2e-test resourceVersion: "22086" uid: f170b43a-c316-466d-a90c-1d8c8d9ce04f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq to ip-10-0-129-35.ec2.internal metadata: creationTimestamp: "2026-07-20T10:22:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:22:45Z" name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq.18c3f85f58e6c11c namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: 9d4b6b7d-599d-483c-ba58-841c2123f3d5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cc9bcc47d-zhtvs source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:45Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: f170b43a-c316-466d-a90c-1d8c8d9ce04f kind: Event lastTimestamp: "2026-07-20T10:22:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-runtime-35b0f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:22:45Z" name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq.18c3f85f62fa9db1 namespace: kserve-ci-e2e-test resourceVersion: "22103" uid: d6d9f452-0b4a-48ca-bf2b-7ee033a50873 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:46Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq namespace: kserve-ci-e2e-test resourceVersion: "22096" uid: f170b43a-c316-466d-a90c-1d8c8d9ce04f kind: Event lastTimestamp: "2026-07-20T10:22:46Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:22:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:22:46Z" name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq.18c3f85f973a53f9 namespace: kserve-ci-e2e-test resourceVersion: "22117" uid: e83062b5-115e-4d87-b10a-4342b580dc66 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: f170b43a-c316-466d-a90c-1d8c8d9ce04f kind: Event lastTimestamp: "2026-07-20T10:22:46Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:038cd29f3e031cdfd5fa54ba896bbecc69cd255eda6b470c2f7bce5282357bb8" already present on machine metadata: creationTimestamp: "2026-07-20T10:22:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:22:46Z" name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq.18c3f85f98f4f3e0 namespace: kserve-ci-e2e-test resourceVersion: "22119" uid: cd6ade26-d738-44fb-bf84-bd4322cf8bcd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: f170b43a-c316-466d-a90c-1d8c8d9ce04f kind: Event lastTimestamp: "2026-07-20T10:22:46Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-20T10:22:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:22:46Z" name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq.18c3f85f9c06f1bd namespace: kserve-ci-e2e-test resourceVersion: "22120" uid: dd909727-845f-449f-a8db-ed3bd06b3f0a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: f170b43a-c316-466d-a90c-1d8c8d9ce04f kind: Event lastTimestamp: "2026-07-20T10:22:46Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-20T10:22:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:22:46Z" name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq.18c3f85f9c81a385 namespace: kserve-ci-e2e-test resourceVersion: "22121" uid: b5e37303-4b2d-4e75-ab64-9203a8078287 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: f170b43a-c316-466d-a90c-1d8c8d9ce04f kind: Event lastTimestamp: "2026-07-20T10:22:51Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:22:51Z" name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq.18c3f860b4d9a123 namespace: kserve-ci-e2e-test resourceVersion: "22159" uid: 675f6eb2-d85a-44de-8748-8552b412b656 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: f170b43a-c316-466d-a90c-1d8c8d9ce04f kind: Event lastTimestamp: "2026-07-20T10:22:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:22:51Z" name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq.18c3f860b9024c58 namespace: kserve-ci-e2e-test resourceVersion: "22163" uid: 5d1c2d97-d40f-4c47-9cf2-a148b3355e24 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: f170b43a-c316-466d-a90c-1d8c8d9ce04f kind: Event lastTimestamp: "2026-07-20T10:22:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:22:51Z" name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq.18c3f860b98637a4 namespace: kserve-ci-e2e-test resourceVersion: "22165" uid: af4b7f81-8000-4383-ba6c-84b7522d03a2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: f170b43a-c316-466d-a90c-1d8c8d9ce04f kind: Event lastTimestamp: "2026-07-20T10:22:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:22:51Z" name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq.18c3f860b990f395 namespace: kserve-ci-e2e-test resourceVersion: "22166" uid: 76be07fd-0642-4602-ad60-fb9c02b69b48 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: f170b43a-c316-466d-a90c-1d8c8d9ce04f kind: Event lastTimestamp: "2026-07-20T10:22:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:22:51Z" name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq.18c3f860bceb9fc4 namespace: kserve-ci-e2e-test resourceVersion: "22169" uid: d35d8469-bdc5-4406-b283-d703bed949b6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: f170b43a-c316-466d-a90c-1d8c8d9ce04f kind: Event lastTimestamp: "2026-07-20T10:22:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:22:51Z" name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq.18c3f860bd6f60e5 namespace: kserve-ci-e2e-test resourceVersion: "22170" uid: ec72f148-b1bb-425b-9632-0096d69cc115 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-20T10:22:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: f170b43a-c316-466d-a90c-1d8c8d9ce04f kind: Event lastTimestamp: "2026-07-20T10:24:01Z" message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:22:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:24:01Z" name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq.18c3f8612cbfc1ea namespace: kserve-ci-e2e-test resourceVersion: "22663" uid: 24e0564c-36d1-437b-b30d-52be7cfa3ba8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:24:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: f170b43a-c316-466d-a90c-1d8c8d9ce04f kind: Event lastTimestamp: "2026-07-20T10:24:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:24:15Z" name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq.18c3f87455e170cb namespace: kserve-ci-e2e-test resourceVersion: "22790" uid: e1a0d7fc-aff0-473e-b15f-cde980cd2895 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:24:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: f170b43a-c316-466d-a90c-1d8c8d9ce04f kind: Event lastTimestamp: "2026-07-20T10:24:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:24:15Z" name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq.18c3f87455e6e513 namespace: kserve-ci-e2e-test resourceVersion: "22791" uid: 98b41b52-a6a8-4dce-b8d4-392612122cf8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:24:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: f170b43a-c316-466d-a90c-1d8c8d9ce04f kind: Event lastTimestamp: "2026-07-20T10:24:19Z" message: 'Readiness probe failed: Get "https://10.133.0.41:8643/healthz": dial tcp 10.133.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:24:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:24:19Z" name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq.18c3f87532ff3c46 namespace: kserve-ci-e2e-test resourceVersion: "22822" uid: db5e93c9-325d-4aaf-955b-dd3d401e794a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-35.ec2.internal source: component: kubelet host: ip-10-0-129-35.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b namespace: kserve-ci-e2e-test resourceVersion: "22074" uid: 9e72ad3a-c8c3-4ca7-afae-705c5f7cd1c7 kind: Event lastTimestamp: "2026-07-20T10:22:45Z" message: 'Created pod: raw-sklearn-runtime-35b0f-predictor-84b5c8555b-648dq' metadata: creationTimestamp: "2026-07-20T10:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:22:45Z" name: raw-sklearn-runtime-35b0f-predictor-84b5c8555b.18c3f85f587b1821 namespace: kserve-ci-e2e-test resourceVersion: "22087" uid: 96ae4d95-68bc-4053-870b-ae4b4db27806 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-35b0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "22072" uid: 06db50d3-555c-472c-92ce-7361cfb0555b kind: Event lastTimestamp: "2026-07-20T10:22:45Z" message: Scaled up replica set raw-sklearn-runtime-35b0f-predictor-84b5c8555b from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:22:45Z" name: raw-sklearn-runtime-35b0f-predictor.18c3f85f539715eb namespace: kserve-ci-e2e-test resourceVersion: "22076" uid: f1c8db12-1fd7-44ca-a5de-d42fb8d92e21 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:23:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-35b0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "22073" uid: 810e72bb-2b02-494d-b86d-afe1f8133e01 kind: Event lastTimestamp: "2026-07-20T10:23:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:23:30Z" name: raw-sklearn-runtime-35b0f-predictor.18c3f862d2c531c5 namespace: kserve-ci-e2e-test resourceVersion: "22449" uid: 0f4b5207-51b3-4015-8070-16994fdc738b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:23:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-35b0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "22073" uid: 810e72bb-2b02-494d-b86d-afe1f8133e01 kind: Event lastTimestamp: "2026-07-20T10:23:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:23:30Z" name: raw-sklearn-runtime-35b0f-predictor.18c3f862d328cbc4 namespace: kserve-ci-e2e-test resourceVersion: "22450" uid: 78e89865-0532-44cf-b0e5-18f6ecc3bec7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:23:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-35b0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "22238" uid: 810e72bb-2b02-494d-b86d-afe1f8133e01 kind: Event lastTimestamp: "2026-07-20T10:24:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:24:15Z" name: raw-sklearn-runtime-35b0f-predictor.18c3f866520e7b66 namespace: kserve-ci-e2e-test resourceVersion: "22773" uid: f21e060a-cc27-4881-8423-0b8e2c399f7c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:23:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-35b0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "22238" uid: 810e72bb-2b02-494d-b86d-afe1f8133e01 kind: Event lastTimestamp: "2026-07-20T10:24:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:24:15Z" name: raw-sklearn-runtime-35b0f-predictor.18c3f866527762bd namespace: kserve-ci-e2e-test resourceVersion: "22775" uid: 21120577-6355-43ab-8559-0d2efa35c4db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-20T10:22:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-35b0f namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 619ff6ae-c741-4aa0-b0ba-5054ee3a74c0 kind: Event lastTimestamp: "2026-07-20T10:24:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:24:11Z" name: raw-sklearn-runtime-35b0f.18c3f85f53954f2c namespace: kserve-ci-e2e-test resourceVersion: "22747" uid: 8ebf8356-d17d-4513-932c-eb99e4f486e2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:22:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-35b0f namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 619ff6ae-c741-4aa0-b0ba-5054ee3a74c0 kind: Event lastTimestamp: "2026-07-20T10:22:45Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-35b0f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-35b0f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:22:45Z" name: raw-sklearn-runtime-35b0f.18c3f85f541b5456 namespace: kserve-ci-e2e-test resourceVersion: "22094" uid: 3716e906-4ff7-4a90-88dc-e1ba9322bec3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-35b0f namespace: kserve-ci-e2e-test resourceVersion: "22069" uid: 619ff6ae-c741-4aa0-b0ba-5054ee3a74c0 kind: Event lastTimestamp: "2026-07-20T10:22:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-35b0f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:22:45Z" name: raw-sklearn-runtime-35b0f.18c3f85f59e9ea7b namespace: kserve-ci-e2e-test resourceVersion: "22097" uid: 14800977-190a-40b9-a02d-c506139810cb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:24:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-35b0f namespace: kserve-ci-e2e-test resourceVersion: "22745" uid: 619ff6ae-c741-4aa0-b0ba-5054ee3a74c0 kind: Event lastTimestamp: "2026-07-20T10:24:11Z" message: InferenceService [raw-sklearn-runtime-35b0f] is Ready metadata: creationTimestamp: "2026-07-20T10:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:24:11Z" name: raw-sklearn-runtime-35b0f.18c3f87353f47b24 namespace: kserve-ci-e2e-test resourceVersion: "22746" uid: 612f4035-c7f1-4481-8e74-98f97aef7072 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "23036"