--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-08T07:19:19.102035Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-e84408-predictor-969d565-fvmzm namespace: kserve-ci-e2e-test resourceVersion: "21436" uid: d0d868cb-5446-43c1-ae77-91b85dff5a53 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-e84408-predictor-969d565-fvmzm to ip-10-0-135-211.ec2.internal metadata: creationTimestamp: "2026-07-08T07:19:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T07:19:19Z" name: isvc-init-fail-e84408-predictor-969d565-fvmzm.18c03f65d8e8445d namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 1ed51a4f-33ca-4309-ba5a-5a39f2ac9e20 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fb64cb696-6mkqx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:19Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-e84408-predictor-969d565-fvmzm namespace: kserve-ci-e2e-test resourceVersion: "21446" uid: d0d868cb-5446-43c1-ae77-91b85dff5a53 kind: Event lastTimestamp: "2026-07-08T07:19:19Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T07:19:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T07:19:19Z" name: isvc-init-fail-e84408-predictor-969d565-fvmzm.18c03f65f26461f5 namespace: kserve-ci-e2e-test resourceVersion: "21455" uid: d2d25ab5-f405-4768-8de1-3a177d7bfe65 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:19:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-e84408-predictor-969d565-fvmzm namespace: kserve-ci-e2e-test resourceVersion: "21438" uid: d0d868cb-5446-43c1-ae77-91b85dff5a53 kind: Event lastTimestamp: "2026-07-08T07:19:24Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773" already present on machine metadata: creationTimestamp: "2026-07-08T07:19:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:19:24Z" name: isvc-init-fail-e84408-predictor-969d565-fvmzm.18c03f65f3db4729 namespace: kserve-ci-e2e-test resourceVersion: "21496" uid: 465fa637-99d7-43b9-851b-b13c88de7e4d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:19:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-e84408-predictor-969d565-fvmzm namespace: kserve-ci-e2e-test resourceVersion: "21438" uid: d0d868cb-5446-43c1-ae77-91b85dff5a53 kind: Event lastTimestamp: "2026-07-08T07:19:24Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T07:19:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:19:24Z" name: isvc-init-fail-e84408-predictor-969d565-fvmzm.18c03f65f64782e7 namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: 8d523d08-71c1-49fc-b42a-b1734c292e68 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:19:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-e84408-predictor-969d565-fvmzm namespace: kserve-ci-e2e-test resourceVersion: "21438" uid: d0d868cb-5446-43c1-ae77-91b85dff5a53 kind: Event lastTimestamp: "2026-07-08T07:19:24Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T07:19:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:19:24Z" name: isvc-init-fail-e84408-predictor-969d565-fvmzm.18c03f65f6b805ae namespace: kserve-ci-e2e-test resourceVersion: "21501" uid: 15365422-8a0d-4b49-99fd-0382d9c5f491 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-e84408-predictor-969d565-fvmzm namespace: kserve-ci-e2e-test resourceVersion: "21438" uid: d0d868cb-5446-43c1-ae77-91b85dff5a53 kind: Event lastTimestamp: "2026-07-08T07:19:29Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-07-08T07:19:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:19:29Z" name: isvc-init-fail-e84408-predictor-969d565-fvmzm.18c03f682df844af namespace: kserve-ci-e2e-test resourceVersion: "21557" uid: 7542b3b4-faca-4912-9f6b-4f583d0cbabd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-e84408-predictor-969d565 namespace: kserve-ci-e2e-test resourceVersion: "21428" uid: 18997e46-cbfa-4c64-bdcd-14bc2760723f kind: Event lastTimestamp: "2026-07-08T07:19:19Z" message: 'Created pod: isvc-init-fail-e84408-predictor-969d565-fvmzm' metadata: creationTimestamp: "2026-07-08T07:19:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:19:19Z" name: isvc-init-fail-e84408-predictor-969d565.18c03f65d87d3938 namespace: kserve-ci-e2e-test resourceVersion: "21437" uid: 260b5545-9480-4618-b55b-f5dec0da96b1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-e84408-predictor namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: c13e398b-2d93-496a-aaa8-f20cdbf3bdfc kind: Event lastTimestamp: "2026-07-08T07:19:19Z" message: Scaled up replica set isvc-init-fail-e84408-predictor-969d565 from 0 to 1 metadata: creationTimestamp: "2026-07-08T07:19:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:19:19Z" name: isvc-init-fail-e84408-predictor.18c03f65d5cf4f92 namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: e23adf45-2728-4968-a805-b7eb1160c78f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T07:19:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-e84408 namespace: kserve-ci-e2e-test resourceVersion: "21418" uid: d8a90c30-7c09-4895-8b7d-39b42033c1df kind: Event lastTimestamp: "2026-07-08T07:19:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T07:19:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:19:24Z" name: isvc-init-fail-e84408.18c03f65d5d918b6 namespace: kserve-ci-e2e-test resourceVersion: "21504" uid: 79a98ee2-27f1-484f-a3d7-f352567b9fff reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-e84408 namespace: kserve-ci-e2e-test resourceVersion: "21418" uid: d8a90c30-7c09-4895-8b7d-39b42033c1df kind: Event lastTimestamp: "2026-07-08T07:19:19Z" message: 'Failed to update status for InferenceService "isvc-init-fail-e84408": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-e84408": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T07:19:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:19:19Z" name: isvc-init-fail-e84408.18c03f65d66227a4 namespace: kserve-ci-e2e-test resourceVersion: "21432" uid: ccbca63a-7202-42ae-b574-530e136bcd05 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-e84408 namespace: kserve-ci-e2e-test resourceVersion: "21418" uid: d8a90c30-7c09-4895-8b7d-39b42033c1df kind: Event lastTimestamp: "2026-07-08T07:19:19Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-e84408": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T07:19:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:19:19Z" name: isvc-init-fail-e84408.18c03f65d66287ec namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: 8c196640-9d69-43c5-b67b-7c9593c72128 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-08T07:13:17.107061Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18475" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp to ip-10-0-135-211.ec2.internal metadata: creationTimestamp: "2026-07-08T07:13:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T07:13:17Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f1190530d1a namespace: kserve-ci-e2e-test resourceVersion: "18479" uid: bb8c5964-f9a1-44bf-9c5c-a797a3ff874a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fb64cb696-6mkqx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:13:17Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: "2026-07-08T07:13:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-raw-53e83-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T07:13:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:13:17Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f119df89fd1 namespace: kserve-ci-e2e-test resourceVersion: "18489" uid: 620eb914-e419-4b0b-aef8-367f42854531 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:13:18Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18483" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: "2026-07-08T07:13:18Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T07:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T07:13:18Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f11cdcb0f92 namespace: kserve-ci-e2e-test resourceVersion: "18498" uid: af5b9f4f-205e-4082-8477-7ef75dafc4a3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:13:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: "2026-07-08T07:13:18Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773" already present on machine metadata: creationTimestamp: "2026-07-08T07:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:13:18Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f11cf4e8e09 namespace: kserve-ci-e2e-test resourceVersion: "18500" uid: 4740d316-66f8-4b8e-a599-bf1e29f52b71 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:13:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: "2026-07-08T07:13:18Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T07:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:13:18Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f11d2608f49 namespace: kserve-ci-e2e-test resourceVersion: "18501" uid: 79e65d79-a380-4807-84d7-b764ad6325a2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:13:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: "2026-07-08T07:13:18Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T07:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:13:18Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f11d2d6d98b namespace: kserve-ci-e2e-test resourceVersion: "18502" uid: 182a4733-47b5-4b0d-8287-520bdf1abc86 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:13:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: "2026-07-08T07:13:22Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-08T07:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:13:22Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f12ba48574f namespace: kserve-ci-e2e-test resourceVersion: "18538" uid: 7819f93f-4178-47b7-bd8e-0ff637776165 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:13:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: "2026-07-08T07:13:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T07:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:13:22Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f12bd2c1339 namespace: kserve-ci-e2e-test resourceVersion: "18541" uid: 9c90b3dc-b7cf-4a0e-8e3e-4beab6348a3d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:13:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: "2026-07-08T07:13:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T07:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:13:22Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f12bda22617 namespace: kserve-ci-e2e-test resourceVersion: "18542" uid: bb963b28-3c85-4c50-bab7-f331e34f79e3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:13:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: "2026-07-08T07:13:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T07:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:13:22Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f12bdabf6c5 namespace: kserve-ci-e2e-test resourceVersion: "18544" uid: c437de82-d59c-46d0-bf53-683d70442802 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:13:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: "2026-07-08T07:13:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T07:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:13:22Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f12c0614284 namespace: kserve-ci-e2e-test resourceVersion: "18546" uid: 6965fcc2-9e07-414c-91ed-c33798d0d0e8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:13:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: "2026-07-08T07:13:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:13:22Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f12c0f17c36 namespace: kserve-ci-e2e-test resourceVersion: "18548" uid: a28af93f-bf52-4a95-b789-8a63387f0288 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:13:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: "2026-07-08T07:13:22Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:3dc370003f480e1124221978de2b776562d1a020a589a2be6d378373cf7b9fa1" already present on machine metadata: creationTimestamp: "2026-07-08T07:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:13:22Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f12c0fa440b namespace: kserve-ci-e2e-test resourceVersion: "18549" uid: fcb81a65-a91f-444a-a9d9-814b4f3baf1a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:13:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: "2026-07-08T07:13:22Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-08T07:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:13:22Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f12c4119bae namespace: kserve-ci-e2e-test resourceVersion: "18551" uid: 192c0b42-743f-459e-abc4-bd9bcc6478e6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:13:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: "2026-07-08T07:13:22Z" message: Started container agent metadata: creationTimestamp: "2026-07-08T07:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:13:22Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f12c48be807 namespace: kserve-ci-e2e-test resourceVersion: "18552" uid: e6add52a-702a-48d0-81f3-c2e60fc2f875 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-08T07:13:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: "2026-07-08T07:15:00Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T07:13:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:15:00Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f12f6509662 namespace: kserve-ci-e2e-test resourceVersion: "19374" uid: bea15322-b05f-4127-8ca3-49088460ae1f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T07:13:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: "2026-07-08T07:15:00Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T07:13:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:15:00Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f13322e2516 namespace: kserve-ci-e2e-test resourceVersion: "19375" uid: 757e3abb-d61c-42d0-8a29-19f4600dbd6f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:14:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: "2026-07-08T07:14:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T07:14:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:14:52Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f27bfb23972 namespace: kserve-ci-e2e-test resourceVersion: "19243" uid: 0cd49678-3480-4689-8dd5-95979974584f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:14:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: "2026-07-08T07:14:52Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-08T07:14:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:14:52Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f27bfb299ee namespace: kserve-ci-e2e-test resourceVersion: "19246" uid: 31047e5b-162c-4ba6-b9d0-4bc60c7fbdd6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:14:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: "2026-07-08T07:14:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:14:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:14:52Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f27bfb2f691 namespace: kserve-ci-e2e-test resourceVersion: "19249" uid: 89a99809-93b7-40c5-8cb2-06de3a407093 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T07:14:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: 3dce2faa-25c2-4a73-9fc2-75885d71fd8e kind: Event lastTimestamp: "2026-07-08T07:15:05Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T07:14:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:15:05Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp.18c03f28626dc9c5 namespace: kserve-ci-e2e-test resourceVersion: "19411" uid: af99a21e-1fdd-441e-81c7-80fa5b22e468 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:13:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-53e83-predictor-6f4c9d7d48 namespace: kserve-ci-e2e-test resourceVersion: "18463" uid: 671dfd06-607f-40f7-9d61-bc57d2ea036b kind: Event lastTimestamp: "2026-07-08T07:13:17Z" message: 'Created pod: isvc-logger-raw-53e83-predictor-6f4c9d7d48-xt2lp' metadata: creationTimestamp: "2026-07-08T07:13:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:13:17Z" name: isvc-logger-raw-53e83-predictor-6f4c9d7d48.18c03f118fd2cacd namespace: kserve-ci-e2e-test resourceVersion: "18476" uid: c687a102-5b8e-456a-a597-e8d121711dcf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:13:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-53e83-predictor namespace: kserve-ci-e2e-test resourceVersion: "18461" uid: 81a895a6-b57b-4209-94f0-74f128c97802 kind: Event lastTimestamp: "2026-07-08T07:13:17Z" message: Scaled up replica set isvc-logger-raw-53e83-predictor-6f4c9d7d48 from 0 to 1 metadata: creationTimestamp: "2026-07-08T07:13:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:13:17Z" name: isvc-logger-raw-53e83-predictor.18c03f118b4b1a9d namespace: kserve-ci-e2e-test resourceVersion: "18464" uid: d9f0b02a-cce0-4fc3-b082-8e98652a5484 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:13:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-53e83-predictor namespace: kserve-ci-e2e-test resourceVersion: "18462" uid: 27319d18-3d73-4662-944e-f9e18cca5da8 kind: Event lastTimestamp: "2026-07-08T07:14:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:13:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:14:02Z" name: isvc-logger-raw-53e83-predictor.18c03f1509c88598 namespace: kserve-ci-e2e-test resourceVersion: "18853" uid: 9901b3ec-56be-4d8d-b288-4343b0ee0665 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:13:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-53e83-predictor namespace: kserve-ci-e2e-test resourceVersion: "18462" uid: 27319d18-3d73-4662-944e-f9e18cca5da8 kind: Event lastTimestamp: "2026-07-08T07:14:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:13:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:14:02Z" name: isvc-logger-raw-53e83-predictor.18c03f150a37c1d9 namespace: kserve-ci-e2e-test resourceVersion: "18854" uid: 54eeeba9-11e0-44a9-a961-8c4e9f973f35 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T07:13:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-53e83-predictor namespace: kserve-ci-e2e-test resourceVersion: "18635" uid: 27319d18-3d73-4662-944e-f9e18cca5da8 kind: Event lastTimestamp: "2026-07-08T07:14:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T07:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:14:47Z" name: isvc-logger-raw-53e83-predictor.18c03f18896229af namespace: kserve-ci-e2e-test resourceVersion: "19178" uid: a7dea5c3-ba6f-4852-b5d6-eb5d4710fd86 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T07:13:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-53e83-predictor namespace: kserve-ci-e2e-test resourceVersion: "18635" uid: 27319d18-3d73-4662-944e-f9e18cca5da8 kind: Event lastTimestamp: "2026-07-08T07:14:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T07:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:14:47Z" name: isvc-logger-raw-53e83-predictor.18c03f1889cee0f6 namespace: kserve-ci-e2e-test resourceVersion: "19179" uid: 6afa0d06-9ea9-41fb-9324-8c6d037cd222 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-08T07:13:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-53e83 namespace: kserve-ci-e2e-test resourceVersion: "18454" uid: 2c60fff6-6e60-43e2-9b31-4566822cc1ca kind: Event lastTimestamp: "2026-07-08T07:14:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T07:13:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:14:40Z" name: isvc-logger-raw-53e83.18c03f118b509345 namespace: kserve-ci-e2e-test resourceVersion: "19126" uid: 5f60b4da-8e5e-49d1-8755-04352bb8c899 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:13:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-53e83 namespace: kserve-ci-e2e-test resourceVersion: "18454" uid: 2c60fff6-6e60-43e2-9b31-4566822cc1ca kind: Event lastTimestamp: "2026-07-08T07:13:17Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-53e83": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-53e83": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T07:13:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:13:17Z" name: isvc-logger-raw-53e83.18c03f118c185bb0 namespace: kserve-ci-e2e-test resourceVersion: "18485" uid: bd828e77-da27-4446-a45c-61818e66e3e8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:13:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-53e83 namespace: kserve-ci-e2e-test resourceVersion: "18455" uid: 2c60fff6-6e60-43e2-9b31-4566822cc1ca kind: Event lastTimestamp: "2026-07-08T07:13:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-53e83": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T07:13:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:13:17Z" name: isvc-logger-raw-53e83.18c03f1192e91676 namespace: kserve-ci-e2e-test resourceVersion: "18486" uid: 9cc7a85d-c129-4013-9c3c-fe3129e23535 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:14:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-53e83 namespace: kserve-ci-e2e-test resourceVersion: "19123" uid: 2c60fff6-6e60-43e2-9b31-4566822cc1ca kind: Event lastTimestamp: "2026-07-08T07:14:40Z" message: InferenceService [isvc-logger-raw-53e83] is Ready metadata: creationTimestamp: "2026-07-08T07:14:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:14:40Z" name: isvc-logger-raw-53e83.18c03f24eb5ab777 namespace: kserve-ci-e2e-test resourceVersion: "19125" uid: aeb9611d-b2db-4683-b686-311a374375bd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T07:17:37.728402Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw namespace: kserve-ci-e2e-test resourceVersion: "20535" uid: d1397740-8702-41ff-a059-a667771fde92 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw to ip-10-0-135-211.ec2.internal metadata: creationTimestamp: "2026-07-08T07:17:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T07:17:37Z" name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw.18c03f4e3e911eec namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: e29875e7-f50b-4f43-be44-c506b5322c6e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fb64cb696-6mkqx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:17:38Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw namespace: kserve-ci-e2e-test resourceVersion: "20543" uid: d1397740-8702-41ff-a059-a667771fde92 kind: Event lastTimestamp: "2026-07-08T07:17:38Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T07:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T07:17:38Z" name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw.18c03f4e57f0e92d namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: 2625b157-c689-41aa-8094-61818babbb2f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw namespace: kserve-ci-e2e-test resourceVersion: "20537" uid: d1397740-8702-41ff-a059-a667771fde92 kind: Event lastTimestamp: "2026-07-08T07:17:38Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773" already present on machine metadata: creationTimestamp: "2026-07-08T07:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:17:38Z" name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw.18c03f4e59643bdb namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: 7de03742-4f04-4d40-a4fa-79a60d1c56fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw namespace: kserve-ci-e2e-test resourceVersion: "20537" uid: d1397740-8702-41ff-a059-a667771fde92 kind: Event lastTimestamp: "2026-07-08T07:17:38Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T07:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:17:38Z" name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw.18c03f4e5c327db4 namespace: kserve-ci-e2e-test resourceVersion: "20557" uid: 0aa4a7e4-d933-48e4-a356-a6037e20f3e0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw namespace: kserve-ci-e2e-test resourceVersion: "20537" uid: d1397740-8702-41ff-a059-a667771fde92 kind: Event lastTimestamp: "2026-07-08T07:17:38Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T07:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:17:38Z" name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw.18c03f4e5ca4ccf8 namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: 0cba553d-a3f8-4ff5-aee6-6918df0415fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:17:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw namespace: kserve-ci-e2e-test resourceVersion: "20537" uid: d1397740-8702-41ff-a059-a667771fde92 kind: Event lastTimestamp: "2026-07-08T07:17:42Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-08T07:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:17:42Z" name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw.18c03f4f4927936a namespace: kserve-ci-e2e-test resourceVersion: "20592" uid: d4b064d5-c830-4a2d-b7e0-cb89ab6b2bde reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:17:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw namespace: kserve-ci-e2e-test resourceVersion: "20537" uid: d1397740-8702-41ff-a059-a667771fde92 kind: Event lastTimestamp: "2026-07-08T07:17:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T07:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:17:42Z" name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw.18c03f4f4bfd77e7 namespace: kserve-ci-e2e-test resourceVersion: "20597" uid: 78c8708b-9e35-4ca7-a051-44cd2f785a03 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:17:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw namespace: kserve-ci-e2e-test resourceVersion: "20537" uid: d1397740-8702-41ff-a059-a667771fde92 kind: Event lastTimestamp: "2026-07-08T07:17:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T07:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:17:42Z" name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw.18c03f4f4c6e902b namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 6fb11403-bc70-46b9-94da-67457fdbe652 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:17:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw namespace: kserve-ci-e2e-test resourceVersion: "20537" uid: d1397740-8702-41ff-a059-a667771fde92 kind: Event lastTimestamp: "2026-07-08T07:17:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T07:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:17:42Z" name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw.18c03f4f4c77b988 namespace: kserve-ci-e2e-test resourceVersion: "20599" uid: bdb09c89-70d1-48fc-8435-7757eaa0056a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:17:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw namespace: kserve-ci-e2e-test resourceVersion: "20537" uid: d1397740-8702-41ff-a059-a667771fde92 kind: Event lastTimestamp: "2026-07-08T07:17:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T07:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:17:42Z" name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw.18c03f4f4f59c147 namespace: kserve-ci-e2e-test resourceVersion: "20602" uid: 119ce9bf-aaa0-4d4a-b8a7-17cbb7d1e42d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:17:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw namespace: kserve-ci-e2e-test resourceVersion: "20537" uid: d1397740-8702-41ff-a059-a667771fde92 kind: Event lastTimestamp: "2026-07-08T07:17:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:17:42Z" name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw.18c03f4f4fc9c6f8 namespace: kserve-ci-e2e-test resourceVersion: "20604" uid: 59f10bd4-0d23-477c-870a-d64a0990fe08 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-08T07:17:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw namespace: kserve-ci-e2e-test resourceVersion: "20537" uid: d1397740-8702-41ff-a059-a667771fde92 kind: Event lastTimestamp: "2026-07-08T07:18:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T07:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:18:49Z" name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw.18c03f4f8553fbfb namespace: kserve-ci-e2e-test resourceVersion: "21082" uid: d872d98a-cdb9-441e-9d45-54b9009a5df9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw namespace: kserve-ci-e2e-test resourceVersion: "20537" uid: d1397740-8702-41ff-a059-a667771fde92 kind: Event lastTimestamp: "2026-07-08T07:19:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T07:19:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:19:13Z" name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw.18c03f64a6e9a7f8 namespace: kserve-ci-e2e-test resourceVersion: "21360" uid: 5097b4a9-cbe7-4947-b048-ca92533f6e44 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw namespace: kserve-ci-e2e-test resourceVersion: "20537" uid: d1397740-8702-41ff-a059-a667771fde92 kind: Event lastTimestamp: "2026-07-08T07:19:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:19:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:19:13Z" name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw.18c03f64a6ecd542 namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: 846a21de-fd1c-4038-adc5-44a6b981a3e3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw namespace: kserve-ci-e2e-test resourceVersion: "20537" uid: d1397740-8702-41ff-a059-a667771fde92 kind: Event lastTimestamp: "2026-07-08T07:19:14Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T07:19:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:19:14Z" name: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw.18c03f64b59d7b79 namespace: kserve-ci-e2e-test resourceVersion: "21366" uid: 64c73955-6971-4060-89e3-412a75251eab reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:17:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-f84f3e-predictor-7bb4d4794 namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: f5f0ada2-87e9-4513-acc2-62d82d5cb952 kind: Event lastTimestamp: "2026-07-08T07:17:37Z" message: 'Created pod: isvc-primary-f84f3e-predictor-7bb4d4794-jfrtw' metadata: creationTimestamp: "2026-07-08T07:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:17:37Z" name: isvc-primary-f84f3e-predictor-7bb4d4794.18c03f4e3e241cdf namespace: kserve-ci-e2e-test resourceVersion: "20536" uid: 3b3feb13-dce7-4ee3-a6d6-e7cf937e51ec reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:17:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-f84f3e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: 0cb5762d-7be8-40be-88a6-6f050ea74055 kind: Event lastTimestamp: "2026-07-08T07:17:37Z" message: Scaled up replica set isvc-primary-f84f3e-predictor-7bb4d4794 from 0 to 1 metadata: creationTimestamp: "2026-07-08T07:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:17:37Z" name: isvc-primary-f84f3e-predictor.18c03f4e395209cf namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: f3f32340-4020-4b48-b267-612177b5f8a6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:17:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-f84f3e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: 5eb29f7e-3279-44e6-9e9a-75e9d8f40be1 kind: Event lastTimestamp: "2026-07-08T07:18:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:18:22Z" name: isvc-primary-f84f3e-predictor.18c03f51b834b299 namespace: kserve-ci-e2e-test resourceVersion: "20894" uid: 8a4a377a-905e-49f2-930d-7c1bc2a82b98 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:17:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-f84f3e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: 5eb29f7e-3279-44e6-9e9a-75e9d8f40be1 kind: Event lastTimestamp: "2026-07-08T07:18:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:18:22Z" name: isvc-primary-f84f3e-predictor.18c03f51b89852a2 namespace: kserve-ci-e2e-test resourceVersion: "20895" uid: 1afa5bde-fe59-43fc-9580-c97525fa9ef6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T07:18:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-f84f3e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20682" uid: 5eb29f7e-3279-44e6-9e9a-75e9d8f40be1 kind: Event lastTimestamp: "2026-07-08T07:19:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T07:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:19:07Z" name: isvc-primary-f84f3e-predictor.18c03f55384d0d9f namespace: kserve-ci-e2e-test resourceVersion: "21280" uid: b8b8b8b3-7bf4-423e-b834-2aecb788d8ae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T07:18:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-f84f3e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20682" uid: 5eb29f7e-3279-44e6-9e9a-75e9d8f40be1 kind: Event lastTimestamp: "2026-07-08T07:19:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T07:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:19:07Z" name: isvc-primary-f84f3e-predictor.18c03f5538b78ab1 namespace: kserve-ci-e2e-test resourceVersion: "21281" uid: 08350996-e5d0-4758-914c-14c8dbf935dd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-08T07:17:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-f84f3e namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: 7e3b0de3-0a6b-4fee-bc52-02844a1f1ebb kind: Event lastTimestamp: "2026-07-08T07:18:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T07:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:18:52Z" name: isvc-primary-f84f3e.18c03f4e39722aff namespace: kserve-ci-e2e-test resourceVersion: "21115" uid: cf61f449-b7f4-482c-8c13-addaeb03a708 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:17:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-f84f3e namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: 7e3b0de3-0a6b-4fee-bc52-02844a1f1ebb kind: Event lastTimestamp: "2026-07-08T07:17:37Z" message: 'Failed to update status for InferenceService "isvc-primary-f84f3e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-f84f3e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T07:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:17:37Z" name: isvc-primary-f84f3e.18c03f4e39e61f78 namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: acf21e79-ad11-4d88-a51b-f4d044c48224 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:17:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-f84f3e namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: 7e3b0de3-0a6b-4fee-bc52-02844a1f1ebb kind: Event lastTimestamp: "2026-07-08T07:17:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-f84f3e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T07:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:17:37Z" name: isvc-primary-f84f3e.18c03f4e39e677aa namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: db6fcf15-91fb-418e-812e-2859a5b25a31 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:18:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-f84f3e namespace: kserve-ci-e2e-test resourceVersion: "21113" uid: 7e3b0de3-0a6b-4fee-bc52-02844a1f1ebb kind: Event lastTimestamp: "2026-07-08T07:18:52Z" message: InferenceService [isvc-primary-f84f3e] is Ready metadata: creationTimestamp: "2026-07-08T07:18:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:18:52Z" name: isvc-primary-f84f3e.18c03f5fbd953a9d namespace: kserve-ci-e2e-test resourceVersion: "21114" uid: d5992f99-ee7c-4e6e-ae00-5ce03393f242 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T07:07:41.381102Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l to ip-10-0-135-211.ec2.internal metadata: creationTimestamp: "2026-07-08T07:07:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T07:07:41Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ec3657f63e1 namespace: kserve-ci-e2e-test resourceVersion: "15391" uid: 65890fac-7705-4c76-8ec7-3f2fb4d6f134 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fb64cb696-6mkqx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:07:41Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:07:41Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T07:07:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T07:07:41Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ec37eba02f5 namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: b61d5e6c-88aa-4dc8-9f77-28b19458a102 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:07:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:07:41Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773" already present on machine metadata: creationTimestamp: "2026-07-08T07:07:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:07:41Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ec3802b660b namespace: kserve-ci-e2e-test resourceVersion: "15411" uid: a20402cf-c626-4166-abef-6936457d6f05 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:07:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:07:41Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T07:07:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:07:41Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ec38303764d namespace: kserve-ci-e2e-test resourceVersion: "15412" uid: 0f390578-1e1a-45ba-9cfd-e1a6ff462cd4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:07:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:07:41Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T07:07:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:07:41Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ec383719fc4 namespace: kserve-ci-e2e-test resourceVersion: "15413" uid: 8dff2552-246c-4a97-8172-5f435b06e2db reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:07:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:07:46Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" metadata: creationTimestamp: "2026-07-08T07:07:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:07:46Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ec49f9fe71d namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: fd858048-4c20-4bce-b182-f49887c235eb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:08:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:08:01Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" in 14.691s (14.691s including waiting). Image size: 1555846056 bytes.' metadata: creationTimestamp: "2026-07-08T07:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:08:01Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ec80b53c2b3 namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: 8dd06ecb-2b97-40f2-a3ea-411fc73e1fce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:08:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:08:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T07:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:08:01Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ec80ed80d7f namespace: kserve-ci-e2e-test resourceVersion: "15570" uid: 0e8d33fc-c9da-4e30-8657-0a265a4ed93d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:08:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:08:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T07:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:08:01Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ec80f4b0e8e namespace: kserve-ci-e2e-test resourceVersion: "15572" uid: 5a09e44c-2485-4d99-a618-97850dac1213 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:08:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:08:01Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-08T07:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:08:01Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ec80f596a12 namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: 1c62b6b3-22e4-452e-88a8-ade446142bdc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:08:03Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.199s (2.199s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-08T07:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:08:03Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ec8927a9ebc namespace: kserve-ci-e2e-test resourceVersion: "15586" uid: 996e29c4-8583-469d-9f66-da6e53830bb5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:08:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T07:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:08:03Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ec89548c652 namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 4d07126e-68b2-4cb0-9a01-5c9bd72444a5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:08:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:08:03Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ec895c07479 namespace: kserve-ci-e2e-test resourceVersion: "15588" uid: 6e689a8e-7c15-4c4d-a088-947dfb2d3650 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:08:03Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:3dc370003f480e1124221978de2b776562d1a020a589a2be6d378373cf7b9fa1" metadata: creationTimestamp: "2026-07-08T07:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:08:03Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ec895ce903e namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: cd8346a2-f2f1-41c0-ab9f-5ea543a5ba67 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:08:07Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:3dc370003f480e1124221978de2b776562d1a020a589a2be6d378373cf7b9fa1" in 3.79s (3.79s including waiting). Image size: 239997027 bytes.' metadata: creationTimestamp: "2026-07-08T07:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:08:07Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ec977b873fb namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 30ed1b83-1a8c-48ac-baaf-7b2ba2ae28fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:08:07Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-08T07:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:08:07Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ec97a897d75 namespace: kserve-ci-e2e-test resourceVersion: "15617" uid: 6a21198d-bde6-49a1-993b-677753348473 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:08:07Z" message: Started container agent metadata: creationTimestamp: "2026-07-08T07:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:08:07Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ec97afa5e8d namespace: kserve-ci-e2e-test resourceVersion: "15618" uid: 414d2c87-da3e-4ff9-8628-9cdcf7b30439 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-08T07:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:09:30Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T07:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:30Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ec989dda7ea namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: a8adda38-5e0b-415f-b116-3b32b04ead70 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T07:08:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:09:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T07:08:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:30Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ec9c5c041da namespace: kserve-ci-e2e-test resourceVersion: "16368" uid: 2f1cc9da-f970-4ba5-8aba-c6d9ef26403b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:09:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T07:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:16Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ed988a2c762 namespace: kserve-ci-e2e-test resourceVersion: "16168" uid: 623e4482-e30a-450b-b9af-104a427023ef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:09:16Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-08T07:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:16Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ed988a300de namespace: kserve-ci-e2e-test resourceVersion: "16173" uid: 669cba51-4c3b-43c1-a792-3c182954e604 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:09:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:16Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03ed988a33eeb namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 9f36f2c2-191e-48f1-8fbf-f88db9c3526a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T07:09:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 99e3d185-f07d-454f-81b8-5252d0d3f566 kind: Event lastTimestamp: "2026-07-08T07:09:33Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T07:09:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:33Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l.18c03eda11fefd91 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: a2a0df47-3bf1-40ea-b978-68b3e919cb05 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:07:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7 namespace: kserve-ci-e2e-test resourceVersion: "15373" uid: 89cba3de-9504-4e0b-b7c9-186c0efca2e7 kind: Event lastTimestamp: "2026-07-08T07:07:41Z" message: 'Created pod: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7-8kj2l' metadata: creationTimestamp: "2026-07-08T07:07:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:07:41Z" name: isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7.18c03ec36440cf6f namespace: kserve-ci-e2e-test resourceVersion: "15388" uid: 1993704d-3649-4550-9aae-7cb6de869ee5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:07:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-0b2da-predictor namespace: kserve-ci-e2e-test resourceVersion: "15372" uid: 32b157a8-7f95-4907-b8f9-70bb5cf1b734 kind: Event lastTimestamp: "2026-07-08T07:07:41Z" message: Scaled up replica set isvc-raw-sklearn-batcher-0b2da-predictor-5cfb8ff6d7 from 0 to 1 metadata: creationTimestamp: "2026-07-08T07:07:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:07:41Z" name: isvc-raw-sklearn-batcher-0b2da-predictor.18c03ec35bae7b6b namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: 821c2f75-bca6-49e8-a577-655e415d34df reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T07:07:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-0b2da-predictor namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: 1474f893-7ef3-4dfc-b915-ed1df1ff7e29 kind: Event lastTimestamp: "2026-07-08T07:08:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:07:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:08:26Z" name: isvc-raw-sklearn-batcher-0b2da-predictor.18c03ec6e0ea3b13 namespace: kserve-ci-e2e-test resourceVersion: "15764" uid: e9b48814-d494-42a0-bf17-d70c65bd8e3f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T07:07:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-0b2da-predictor namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: 1474f893-7ef3-4dfc-b915-ed1df1ff7e29 kind: Event lastTimestamp: "2026-07-08T07:08:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:07:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:08:26Z" name: isvc-raw-sklearn-batcher-0b2da-predictor.18c03ec6e1529ac3 namespace: kserve-ci-e2e-test resourceVersion: "15765" uid: bcda79f0-3e2f-4d41-a60a-e6fe57b83abf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T07:08:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-0b2da-predictor namespace: kserve-ci-e2e-test resourceVersion: "15530" uid: 1474f893-7ef3-4dfc-b915-ed1df1ff7e29 kind: Event lastTimestamp: "2026-07-08T07:09:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T07:08:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:09:11Z" name: isvc-raw-sklearn-batcher-0b2da-predictor.18c03ed15df5d88d namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: a7196ef6-bc40-4c07-bc07-81ba6e484af9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T07:08:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-0b2da-predictor namespace: kserve-ci-e2e-test resourceVersion: "15530" uid: 1474f893-7ef3-4dfc-b915-ed1df1ff7e29 kind: Event lastTimestamp: "2026-07-08T07:09:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T07:08:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:09:11Z" name: isvc-raw-sklearn-batcher-0b2da-predictor.18c03ed15e79cd0d namespace: kserve-ci-e2e-test resourceVersion: "16089" uid: 57333c71-e889-48a0-84c4-798b592f37c0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-08T07:07:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-0b2da namespace: kserve-ci-e2e-test resourceVersion: "15365" uid: 5e65e3be-fe49-4cf9-bff7-97a289d9564f kind: Event lastTimestamp: "2026-07-08T07:09:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T07:07:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:09:10Z" name: isvc-raw-sklearn-batcher-0b2da.18c03ec361c72e42 namespace: kserve-ci-e2e-test resourceVersion: "16083" uid: 71e36bee-28ee-43f3-8994-b75b9477a5ab reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:07:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-0b2da namespace: kserve-ci-e2e-test resourceVersion: "15365" uid: 5e65e3be-fe49-4cf9-bff7-97a289d9564f kind: Event lastTimestamp: "2026-07-08T07:07:41Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-0b2da": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-0b2da": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T07:07:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:07:41Z" name: isvc-raw-sklearn-batcher-0b2da.18c03ec3622d2d1c namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: 7174b7bb-81b1-4da1-a784-ce8e906fba29 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:07:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-0b2da namespace: kserve-ci-e2e-test resourceVersion: "15365" uid: 5e65e3be-fe49-4cf9-bff7-97a289d9564f kind: Event lastTimestamp: "2026-07-08T07:07:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-0b2da": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T07:07:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:07:41Z" name: isvc-raw-sklearn-batcher-0b2da.18c03ec3622d7bdc namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: c11c1016-286d-4661-8508-3c2cced4775e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-0b2da namespace: kserve-ci-e2e-test resourceVersion: "16081" uid: 5e65e3be-fe49-4cf9-bff7-97a289d9564f kind: Event lastTimestamp: "2026-07-08T07:09:10Z" message: InferenceService [isvc-raw-sklearn-batcher-0b2da] is Ready metadata: creationTimestamp: "2026-07-08T07:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:09:10Z" name: isvc-raw-sklearn-batcher-0b2da.18c03ed83c0ab72a namespace: kserve-ci-e2e-test resourceVersion: "16082" uid: 39d13f28-14c2-44da-8ed0-3c6d0e481f09 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T07:18:57.853173Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-f84f3e-predictor-64f457bbbf-8f9zr namespace: kserve-ci-e2e-test resourceVersion: "21171" uid: 2769bb4f-9abd-465e-a9d3-c68dbf3feabf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-f84f3e-predictor-64f457bbbf-8f9zr to ip-10-0-135-211.ec2.internal metadata: creationTimestamp: "2026-07-08T07:18:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T07:18:57Z" name: isvc-secondary-f84f3e-predictor-64f457bbbf-8f9zr.18c03f60e6600f50 namespace: kserve-ci-e2e-test resourceVersion: "21177" uid: c3a6fb7e-1340-4c05-8aaf-213228e68989 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fb64cb696-6mkqx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:18:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-f84f3e-predictor-64f457bbbf-8f9zr namespace: kserve-ci-e2e-test resourceVersion: "21181" uid: 2769bb4f-9abd-465e-a9d3-c68dbf3feabf kind: Event lastTimestamp: "2026-07-08T07:18:58Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T07:18:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T07:18:58Z" name: isvc-secondary-f84f3e-predictor-64f457bbbf-8f9zr.18c03f60ffaa8de8 namespace: kserve-ci-e2e-test resourceVersion: "21189" uid: b0072282-bba8-466d-88fb-e92fa20b7f34 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:18:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-f84f3e-predictor-64f457bbbf-8f9zr namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: 2769bb4f-9abd-465e-a9d3-c68dbf3feabf kind: Event lastTimestamp: "2026-07-08T07:19:03Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773" already present on machine metadata: creationTimestamp: "2026-07-08T07:18:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:19:03Z" name: isvc-secondary-f84f3e-predictor-64f457bbbf-8f9zr.18c03f61010cca51 namespace: kserve-ci-e2e-test resourceVersion: "21239" uid: d36072a1-4b19-49c1-8abf-de7990f0a1a3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:18:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-f84f3e-predictor-64f457bbbf-8f9zr namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: 2769bb4f-9abd-465e-a9d3-c68dbf3feabf kind: Event lastTimestamp: "2026-07-08T07:19:03Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T07:18:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:19:03Z" name: isvc-secondary-f84f3e-predictor-64f457bbbf-8f9zr.18c03f61041e7263 namespace: kserve-ci-e2e-test resourceVersion: "21242" uid: 45052ff3-9350-4ae5-bdc0-1249cb2d0256 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:18:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-f84f3e-predictor-64f457bbbf-8f9zr namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: 2769bb4f-9abd-465e-a9d3-c68dbf3feabf kind: Event lastTimestamp: "2026-07-08T07:19:03Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T07:18:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:19:03Z" name: isvc-secondary-f84f3e-predictor-64f457bbbf-8f9zr.18c03f6104916156 namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: 14c3bd81-8bd6-42b6-b4ba-5e140e6a2532 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-f84f3e-predictor-64f457bbbf-8f9zr namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: 2769bb4f-9abd-465e-a9d3-c68dbf3feabf kind: Event lastTimestamp: "2026-07-08T07:19:06Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-f84f3e-predictor-64f457bbbf-8f9zr_kserve-ci-e2e-test(2769bb4f-9abd-465e-a9d3-c68dbf3feabf) metadata: creationTimestamp: "2026-07-08T07:19:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:19:06Z" name: isvc-secondary-f84f3e-predictor-64f457bbbf-8f9zr.18c03f62eddd8cae namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: c952dac3-321a-4489-aff7-75c4ea19aeb7 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:18:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-f84f3e-predictor-64f457bbbf namespace: kserve-ci-e2e-test resourceVersion: "21162" uid: 891f0075-98be-4d2c-89ac-d21817990db4 kind: Event lastTimestamp: "2026-07-08T07:18:57Z" message: 'Created pod: isvc-secondary-f84f3e-predictor-64f457bbbf-8f9zr' metadata: creationTimestamp: "2026-07-08T07:18:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:18:57Z" name: isvc-secondary-f84f3e-predictor-64f457bbbf.18c03f60e5edaf17 namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: 08c21629-493c-4d00-9359-accc75ff3979 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:18:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-f84f3e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21160" uid: 344e8b5b-682d-4be0-9430-ee0b94832b42 kind: Event lastTimestamp: "2026-07-08T07:18:57Z" message: Scaled up replica set isvc-secondary-f84f3e-predictor-64f457bbbf from 0 to 1 metadata: creationTimestamp: "2026-07-08T07:18:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:18:57Z" name: isvc-secondary-f84f3e-predictor.18c03f60e3012736 namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: 18ab7af0-b77d-42b5-a02b-46b268d10178 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-f84f3e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21161" uid: 53146f4e-8ebf-438e-abc9-f1c64b0aa7f0 kind: Event lastTimestamp: "2026-07-08T07:19:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:19:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:19:12Z" name: isvc-secondary-f84f3e-predictor.18c03f6461747222 namespace: kserve-ci-e2e-test resourceVersion: "21318" uid: 32321a68-0ee1-4c88-a760-cff05d855fee reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-f84f3e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21161" uid: 53146f4e-8ebf-438e-abc9-f1c64b0aa7f0 kind: Event lastTimestamp: "2026-07-08T07:19:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:19:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:19:12Z" name: isvc-secondary-f84f3e-predictor.18c03f6461da96b6 namespace: kserve-ci-e2e-test resourceVersion: "21320" uid: 629badb3-dc70-4d4f-8b7a-ded221493217 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-08T07:18:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-f84f3e namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: 929c39c7-c36b-4288-bd44-6c5ad8460e28 kind: Event lastTimestamp: "2026-07-08T07:19:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T07:18:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:19:06Z" name: isvc-secondary-f84f3e.18c03f60e308d9ee namespace: kserve-ci-e2e-test resourceVersion: "21275" uid: 15923bd9-bc63-4638-93a4-38a446695367 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:18:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-f84f3e namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: 929c39c7-c36b-4288-bd44-6c5ad8460e28 kind: Event lastTimestamp: "2026-07-08T07:18:57Z" message: 'Failed to update status for InferenceService "isvc-secondary-f84f3e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-f84f3e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T07:18:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:18:57Z" name: isvc-secondary-f84f3e.18c03f60e38141e8 namespace: kserve-ci-e2e-test resourceVersion: "21166" uid: 0f53694b-2632-45da-a54b-a7785e0ee7f3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:18:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-f84f3e namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: 929c39c7-c36b-4288-bd44-6c5ad8460e28 kind: Event lastTimestamp: "2026-07-08T07:18:57Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-f84f3e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T07:18:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:18:57Z" name: isvc-secondary-f84f3e.18c03f60e381ad06 namespace: kserve-ci-e2e-test resourceVersion: "21168" uid: 2153ba1c-a132-4b4d-a793-e7da14106735 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-08T07:09:16.468861Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: 252cd5fc-01d6-444b-b394-04708459db4c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm to ip-10-0-135-211.ec2.internal metadata: creationTimestamp: "2026-07-08T07:09:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T07:09:16Z" name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm.18c03ed9892b7990 namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: 0848db1d-d72f-4d3f-a440-a558135847f1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fb64cb696-6mkqx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: 252cd5fc-01d6-444b-b394-04708459db4c kind: Event lastTimestamp: "2026-07-08T07:09:16Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T07:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T07:09:16Z" name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm.18c03ed9a4568cdd namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: 1a189209-c080-4760-8e26-6cb30da6d203 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 252cd5fc-01d6-444b-b394-04708459db4c kind: Event lastTimestamp: "2026-07-08T07:09:16Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773" already present on machine metadata: creationTimestamp: "2026-07-08T07:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:16Z" name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm.18c03ed9a5d74a43 namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 70d96dab-e4cd-4214-a681-ee14bac73f79 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 252cd5fc-01d6-444b-b394-04708459db4c kind: Event lastTimestamp: "2026-07-08T07:09:16Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T07:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:16Z" name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm.18c03ed9a8955ea6 namespace: kserve-ci-e2e-test resourceVersion: "16226" uid: 945b0f37-6a63-4616-b6af-76bd805384de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 252cd5fc-01d6-444b-b394-04708459db4c kind: Event lastTimestamp: "2026-07-08T07:09:17Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T07:09:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:17Z" name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm.18c03ed9a909f5d2 namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: 70e31b27-526d-4267-aab2-49af3c40634a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 252cd5fc-01d6-444b-b394-04708459db4c kind: Event lastTimestamp: "2026-07-08T07:09:21Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-08T07:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:21Z" name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm.18c03eda9ad89f6c namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: a8b0b655-1931-4c62-854a-5bc3bbedb7ea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 252cd5fc-01d6-444b-b394-04708459db4c kind: Event lastTimestamp: "2026-07-08T07:09:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T07:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:21Z" name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm.18c03eda9da53f77 namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 7fdc057d-df58-4b42-b9b8-5cd19b266866 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 252cd5fc-01d6-444b-b394-04708459db4c kind: Event lastTimestamp: "2026-07-08T07:09:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T07:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:21Z" name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm.18c03eda9e371591 namespace: kserve-ci-e2e-test resourceVersion: "16281" uid: 47351ceb-e095-4ccd-b235-66de8b047e58 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 252cd5fc-01d6-444b-b394-04708459db4c kind: Event lastTimestamp: "2026-07-08T07:09:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T07:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:21Z" name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm.18c03eda9e40318f namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: f7ce23d8-e73d-4447-8026-a352a2e956fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 252cd5fc-01d6-444b-b394-04708459db4c kind: Event lastTimestamp: "2026-07-08T07:09:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T07:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:21Z" name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm.18c03edaa1cc5d99 namespace: kserve-ci-e2e-test resourceVersion: "16286" uid: 13b0ded4-e38b-4e21-bae2-c8c9100051b8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 252cd5fc-01d6-444b-b394-04708459db4c kind: Event lastTimestamp: "2026-07-08T07:09:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:21Z" name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm.18c03edaa24dd6a6 namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: c742cd66-e060-423b-a05b-ccf3290c29b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T07:09:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 252cd5fc-01d6-444b-b394-04708459db4c kind: Event lastTimestamp: "2026-07-08T07:11:09Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T07:09:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:09Z" name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm.18c03edb12f2534d namespace: kserve-ci-e2e-test resourceVersion: "17307" uid: 93605d01-7569-4308-bb5a-b198427fdea3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 252cd5fc-01d6-444b-b394-04708459db4c kind: Event lastTimestamp: "2026-07-08T07:11:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T07:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:06Z" name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm.18c03ef333a92884 namespace: kserve-ci-e2e-test resourceVersion: "17194" uid: 3fa42dd0-7c84-48da-acae-d2f393166a2e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 252cd5fc-01d6-444b-b394-04708459db4c kind: Event lastTimestamp: "2026-07-08T07:11:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:06Z" name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm.18c03ef333a9caeb namespace: kserve-ci-e2e-test resourceVersion: "17195" uid: b780a015-40e5-4e4a-95f8-4afbfed4ea50 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 252cd5fc-01d6-444b-b394-04708459db4c kind: Event lastTimestamp: "2026-07-08T07:11:09Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T07:11:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:09Z" name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm.18c03ef3c1662e00 namespace: kserve-ci-e2e-test resourceVersion: "17306" uid: 0496b262-5499-4d71-94eb-166bbbeac363 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47 namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: c396c01a-3bd2-445a-a0d0-89e83c17231b kind: Event lastTimestamp: "2026-07-08T07:09:16Z" message: 'Created pod: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47-mg8lm' metadata: creationTimestamp: "2026-07-08T07:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:09:16Z" name: isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47.18c03ed988a223a5 namespace: kserve-ci-e2e-test resourceVersion: "16169" uid: 31dded3a-fde8-4522-85bd-f8ade53f52d2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-a4509-predictor namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: e16d0e11-efac-44fc-a1cd-b42f35c94794 kind: Event lastTimestamp: "2026-07-08T07:09:16Z" message: Scaled up replica set isvc-sklearn-graph-raw-a4509-predictor-7966d8cf47 from 0 to 1 metadata: creationTimestamp: "2026-07-08T07:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:09:16Z" name: isvc-sklearn-graph-raw-a4509-predictor.18c03ed983bbecf1 namespace: kserve-ci-e2e-test resourceVersion: "16142" uid: dac85cf5-d102-4483-89b2-3ba643fb5b8a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:09:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-a4509-predictor namespace: kserve-ci-e2e-test resourceVersion: "16140" uid: a2a20ef8-ed55-48dd-a9f3-c9e25843bfb8 kind: Event lastTimestamp: "2026-07-08T07:10:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:10:01Z" name: isvc-sklearn-graph-raw-a4509-predictor.18c03edd02634b2f namespace: kserve-ci-e2e-test resourceVersion: "16616" uid: 4a5fd23f-be39-4627-b83a-c7f9d2976f8d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:09:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-a4509-predictor namespace: kserve-ci-e2e-test resourceVersion: "16140" uid: a2a20ef8-ed55-48dd-a9f3-c9e25843bfb8 kind: Event lastTimestamp: "2026-07-08T07:10:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:10:01Z" name: isvc-sklearn-graph-raw-a4509-predictor.18c03edd02ebc18b namespace: kserve-ci-e2e-test resourceVersion: "16617" uid: bf62e253-f4db-49ee-be6d-6fd4d20024bf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T07:09:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-a4509-predictor namespace: kserve-ci-e2e-test resourceVersion: "16375" uid: a2a20ef8-ed55-48dd-a9f3-c9e25843bfb8 kind: Event lastTimestamp: "2026-07-08T07:11:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T07:09:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:11:01Z" name: isvc-sklearn-graph-raw-a4509-predictor.18c03ee0821a0e08 namespace: kserve-ci-e2e-test resourceVersion: "17106" uid: afd079d1-2e6b-42cf-a3a5-c21c91cab7d4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T07:09:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-a4509-predictor namespace: kserve-ci-e2e-test resourceVersion: "16375" uid: a2a20ef8-ed55-48dd-a9f3-c9e25843bfb8 kind: Event lastTimestamp: "2026-07-08T07:11:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T07:09:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:11:01Z" name: isvc-sklearn-graph-raw-a4509-predictor.18c03ee0827f1a4a namespace: kserve-ci-e2e-test resourceVersion: "17107" uid: aa5cf6ac-6603-448e-ae61-952be3ab720d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-08T07:09:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-a4509 namespace: kserve-ci-e2e-test resourceVersion: "16130" uid: 9b906d6e-163b-4231-8ee8-522c3b8a25ea kind: Event lastTimestamp: "2026-07-08T07:10:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T07:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:10:39Z" name: isvc-sklearn-graph-raw-a4509.18c03ed983c6ada0 namespace: kserve-ci-e2e-test resourceVersion: "16904" uid: dbf48338-db17-4fff-8185-6522492edb5b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-a4509 namespace: kserve-ci-e2e-test resourceVersion: "16130" uid: 9b906d6e-163b-4231-8ee8-522c3b8a25ea kind: Event lastTimestamp: "2026-07-08T07:09:16Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-a4509": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-a4509": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T07:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:09:16Z" name: isvc-sklearn-graph-raw-a4509.18c03ed98436a0f9 namespace: kserve-ci-e2e-test resourceVersion: "16145" uid: 368c8a44-1520-4565-81e6-152a4982b2cb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:10:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-a4509 namespace: kserve-ci-e2e-test resourceVersion: "16898" uid: 9b906d6e-163b-4231-8ee8-522c3b8a25ea kind: Event lastTimestamp: "2026-07-08T07:10:39Z" message: InferenceService [isvc-sklearn-graph-raw-a4509] is Ready metadata: creationTimestamp: "2026-07-08T07:10:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:10:39Z" name: isvc-sklearn-graph-raw-a4509.18c03eeccaa39867 namespace: kserve-ci-e2e-test resourceVersion: "16899" uid: 42f48256-a1a2-441e-9ccd-9c0bec9974db reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T07:11:06.765592Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb namespace: kserve-ci-e2e-test resourceVersion: "17202" uid: a93541e4-a124-469f-903b-c22754a2481d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb to ip-10-0-135-211.ec2.internal metadata: creationTimestamp: "2026-07-08T07:11:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T07:11:06Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb.18c03ef3375e07ae namespace: kserve-ci-e2e-test resourceVersion: "17209" uid: 5f506494-5111-4eb6-93b1-4bd4e0dd1840 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fb64cb696-6mkqx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb namespace: kserve-ci-e2e-test resourceVersion: "17222" uid: a93541e4-a124-469f-903b-c22754a2481d kind: Event lastTimestamp: "2026-07-08T07:11:07Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T07:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T07:11:07Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb.18c03ef350858f66 namespace: kserve-ci-e2e-test resourceVersion: "17271" uid: 38927011-365c-4880-a41f-3e088d885f89 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: a93541e4-a124-469f-903b-c22754a2481d kind: Event lastTimestamp: "2026-07-08T07:11:07Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773" already present on machine metadata: creationTimestamp: "2026-07-08T07:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:07Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb.18c03ef3521b4938 namespace: kserve-ci-e2e-test resourceVersion: "17273" uid: dc9b2e47-f976-44de-a8ed-ac305ee1c4b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: a93541e4-a124-469f-903b-c22754a2481d kind: Event lastTimestamp: "2026-07-08T07:11:07Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T07:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:07Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb.18c03ef3552d8115 namespace: kserve-ci-e2e-test resourceVersion: "17276" uid: 0f4a2b4d-2efd-4039-ad12-d2e225bb0cbf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: a93541e4-a124-469f-903b-c22754a2481d kind: Event lastTimestamp: "2026-07-08T07:11:07Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T07:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:07Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb.18c03ef35597e1df namespace: kserve-ci-e2e-test resourceVersion: "17277" uid: 3c904e30-dce9-4b80-933b-c64ad88e2eea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: a93541e4-a124-469f-903b-c22754a2481d kind: Event lastTimestamp: "2026-07-08T07:11:11Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-08T07:11:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:11Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb.18c03ef454caa062 namespace: kserve-ci-e2e-test resourceVersion: "17330" uid: 399cd35e-fb3f-4b4c-bc4f-c415ffbe35ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: a93541e4-a124-469f-903b-c22754a2481d kind: Event lastTimestamp: "2026-07-08T07:11:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T07:11:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:11Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb.18c03ef458ecb727 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: ca24cbd9-6241-4082-af9b-038805f3d9ab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: a93541e4-a124-469f-903b-c22754a2481d kind: Event lastTimestamp: "2026-07-08T07:11:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T07:11:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:11Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb.18c03ef4595d5a7b namespace: kserve-ci-e2e-test resourceVersion: "17337" uid: 00cfbcfc-80cf-4885-9a2e-ec10f517bdc2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: a93541e4-a124-469f-903b-c22754a2481d kind: Event lastTimestamp: "2026-07-08T07:11:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T07:11:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:11Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb.18c03ef459661e46 namespace: kserve-ci-e2e-test resourceVersion: "17338" uid: 86d49eac-29c2-4c20-851a-226be33c8821 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: a93541e4-a124-469f-903b-c22754a2481d kind: Event lastTimestamp: "2026-07-08T07:11:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T07:11:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:11Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb.18c03ef45c3f53af namespace: kserve-ci-e2e-test resourceVersion: "17340" uid: 033d338b-c93f-4106-a6b2-4ab0faba9059 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: a93541e4-a124-469f-903b-c22754a2481d kind: Event lastTimestamp: "2026-07-08T07:11:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:11:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:11Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb.18c03ef45caffaa8 namespace: kserve-ci-e2e-test resourceVersion: "17341" uid: 71b163f4-cca9-4161-811b-a24629de6819 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-08T07:11:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: a93541e4-a124-469f-903b-c22754a2481d kind: Event lastTimestamp: "2026-07-08T07:12:59Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T07:11:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:12:59Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb.18c03ef491186206 namespace: kserve-ci-e2e-test resourceVersion: "18308" uid: baf8b0fe-73b0-4f43-bd21-9148fe9fae77 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: a93541e4-a124-469f-903b-c22754a2481d kind: Event lastTimestamp: "2026-07-08T07:12:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T07:12:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:12:57Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb.18c03f0cea19ba2c namespace: kserve-ci-e2e-test resourceVersion: "18243" uid: a6244331-127b-4a57-91cd-a7d4a949a5d5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: a93541e4-a124-469f-903b-c22754a2481d kind: Event lastTimestamp: "2026-07-08T07:12:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:12:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:12:57Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb.18c03f0cea213104 namespace: kserve-ci-e2e-test resourceVersion: "18249" uid: d98b1edb-acf4-4b70-b4bf-5f81ff7dd9af reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: a93541e4-a124-469f-903b-c22754a2481d kind: Event lastTimestamp: "2026-07-08T07:12:58Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T07:12:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:12:58Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb.18c03f0d3f673da1 namespace: kserve-ci-e2e-test resourceVersion: "18287" uid: 7059e61f-8ec1-453c-8c2d-de6a6cd0549e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79 namespace: kserve-ci-e2e-test resourceVersion: "17183" uid: 206ac312-49b5-4961-aac1-e6bd711712b9 kind: Event lastTimestamp: "2026-07-08T07:11:06Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79-mjzwb' metadata: creationTimestamp: "2026-07-08T07:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:11:06Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79.18c03ef336d51d62 namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 946aa440-c1fe-409e-b275-d5020b20f3e6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-aea5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17176" uid: a9d9a3b2-752b-4ea0-ab2f-6357d950197b kind: Event lastTimestamp: "2026-07-08T07:11:06Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-aea5a-predictor-7b7677b79 from 0 to 1 metadata: creationTimestamp: "2026-07-08T07:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:11:06Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor.18c03ef332907d43 namespace: kserve-ci-e2e-test resourceVersion: "17187" uid: b8d58583-72fa-4671-b42c-d08eb2cac5d4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T07:11:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-aea5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: a17e6bb8-643c-4d12-ac8a-1792827fa8fa kind: Event lastTimestamp: "2026-07-08T07:12:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:12:06Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor.18c03ef6b15058ef namespace: kserve-ci-e2e-test resourceVersion: "17764" uid: 40f5d566-7842-4720-8b06-12b1d039b774 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T07:11:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-aea5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: a17e6bb8-643c-4d12-ac8a-1792827fa8fa kind: Event lastTimestamp: "2026-07-08T07:12:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:12:06Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor.18c03ef6b1b0d096 namespace: kserve-ci-e2e-test resourceVersion: "17765" uid: 2d93be77-19bb-48a1-88d4-75d36af24fe3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T07:12:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-aea5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17437" uid: a17e6bb8-643c-4d12-ac8a-1792827fa8fa kind: Event lastTimestamp: "2026-07-08T07:12:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T07:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:12:51Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor.18c03f04ad6baf79 namespace: kserve-ci-e2e-test resourceVersion: "18139" uid: ebcd8d3d-311b-4c50-bc0d-3c1c319b0082 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T07:12:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-aea5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17437" uid: a17e6bb8-643c-4d12-ac8a-1792827fa8fa kind: Event lastTimestamp: "2026-07-08T07:12:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T07:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:12:51Z" name: isvc-sklearn-graph-raw-hpa-aea5a-predictor.18c03f04ae117e00 namespace: kserve-ci-e2e-test resourceVersion: "18140" uid: 3eff8bb2-4fd0-4004-a40f-61b2a3daf6b7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-08T07:11:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-aea5a namespace: kserve-ci-e2e-test resourceVersion: "17163" uid: 2a199e49-e3c5-4f07-8f4e-c7e9edc01f55 kind: Event lastTimestamp: "2026-07-08T07:12:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T07:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:12:29Z" name: isvc-sklearn-graph-raw-hpa-aea5a.18c03ef332d2eafc namespace: kserve-ci-e2e-test resourceVersion: "17948" uid: 77f02ba6-fc1e-49ab-9039-d885e6795a5c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-aea5a namespace: kserve-ci-e2e-test resourceVersion: "17163" uid: 2a199e49-e3c5-4f07-8f4e-c7e9edc01f55 kind: Event lastTimestamp: "2026-07-08T07:11:06Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-aea5a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-aea5a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T07:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:11:06Z" name: isvc-sklearn-graph-raw-hpa-aea5a.18c03ef3332a4162 namespace: kserve-ci-e2e-test resourceVersion: "17191" uid: b5b1aff9-3093-4759-99f5-e13cf8a5ffb8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-aea5a namespace: kserve-ci-e2e-test resourceVersion: "17945" uid: 2a199e49-e3c5-4f07-8f4e-c7e9edc01f55 kind: Event lastTimestamp: "2026-07-08T07:12:29Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-aea5a] is Ready metadata: creationTimestamp: "2026-07-08T07:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:12:29Z" name: isvc-sklearn-graph-raw-hpa-aea5a.18c03f0690a0fb34 namespace: kserve-ci-e2e-test resourceVersion: "17946" uid: dbaac49e-6601-4d0e-a226-82e28f19b333 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T07:14:52.491165Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp namespace: kserve-ci-e2e-test resourceVersion: "19256" uid: 395bc3c1-2bdc-4dbd-aa3f-82ef64696d25 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp to ip-10-0-135-211.ec2.internal metadata: creationTimestamp: "2026-07-08T07:14:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T07:14:52Z" name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp.18c03f27c5a8fbde namespace: kserve-ci-e2e-test resourceVersion: "19264" uid: 50a77a06-3da3-47e6-9b6a-7515bf09b8d2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fb64cb696-6mkqx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:14:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp namespace: kserve-ci-e2e-test resourceVersion: "19259" uid: 395bc3c1-2bdc-4dbd-aa3f-82ef64696d25 kind: Event lastTimestamp: "2026-07-08T07:14:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-bba5b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T07:14:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:14:52Z" name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp.18c03f27d158e4eb namespace: kserve-ci-e2e-test resourceVersion: "19290" uid: aba1b085-e413-4715-b557-3e00c5001ee0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:14:53Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp namespace: kserve-ci-e2e-test resourceVersion: "19278" uid: 395bc3c1-2bdc-4dbd-aa3f-82ef64696d25 kind: Event lastTimestamp: "2026-07-08T07:14:53Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T07:14:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T07:14:53Z" name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp.18c03f2803b3a800 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 689a4012-0a7d-4dc8-bf84-3e60e414f30d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:14:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp namespace: kserve-ci-e2e-test resourceVersion: "19259" uid: 395bc3c1-2bdc-4dbd-aa3f-82ef64696d25 kind: Event lastTimestamp: "2026-07-08T07:14:53Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773" already present on machine metadata: creationTimestamp: "2026-07-08T07:14:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:14:53Z" name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp.18c03f28050a325f namespace: kserve-ci-e2e-test resourceVersion: "19308" uid: 71e1beaf-fdf9-4de4-ac11-c846437556d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:14:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp namespace: kserve-ci-e2e-test resourceVersion: "19259" uid: 395bc3c1-2bdc-4dbd-aa3f-82ef64696d25 kind: Event lastTimestamp: "2026-07-08T07:14:53Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T07:14:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:14:53Z" name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp.18c03f2807e46d5e namespace: kserve-ci-e2e-test resourceVersion: "19309" uid: 5068a1d1-19f0-4fc3-8d60-6240063534cf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:14:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp namespace: kserve-ci-e2e-test resourceVersion: "19259" uid: 395bc3c1-2bdc-4dbd-aa3f-82ef64696d25 kind: Event lastTimestamp: "2026-07-08T07:14:53Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T07:14:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:14:53Z" name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp.18c03f2808583bd5 namespace: kserve-ci-e2e-test resourceVersion: "19310" uid: e5bec850-c2b8-47bf-a4f8-f2137da5716b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp namespace: kserve-ci-e2e-test resourceVersion: "19259" uid: 395bc3c1-2bdc-4dbd-aa3f-82ef64696d25 kind: Event lastTimestamp: "2026-07-08T07:14:57Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-08T07:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:14:57Z" name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp.18c03f28f1e4abbc namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 16583bcc-3cfd-49fb-8c36-f6919460e88c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp namespace: kserve-ci-e2e-test resourceVersion: "19259" uid: 395bc3c1-2bdc-4dbd-aa3f-82ef64696d25 kind: Event lastTimestamp: "2026-07-08T07:14:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T07:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:14:57Z" name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp.18c03f28f515e8ea namespace: kserve-ci-e2e-test resourceVersion: "19348" uid: a9723a91-cbd8-4dd7-829d-c30cedc6e58f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp namespace: kserve-ci-e2e-test resourceVersion: "19259" uid: 395bc3c1-2bdc-4dbd-aa3f-82ef64696d25 kind: Event lastTimestamp: "2026-07-08T07:14:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T07:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:14:57Z" name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp.18c03f28f5892e03 namespace: kserve-ci-e2e-test resourceVersion: "19351" uid: 51347b53-436a-4df2-ae06-4a7282b629bc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp namespace: kserve-ci-e2e-test resourceVersion: "19259" uid: 395bc3c1-2bdc-4dbd-aa3f-82ef64696d25 kind: Event lastTimestamp: "2026-07-08T07:14:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T07:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:14:57Z" name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp.18c03f28f5922f36 namespace: kserve-ci-e2e-test resourceVersion: "19352" uid: 109c587d-d0b8-4624-9223-5813ed896189 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp namespace: kserve-ci-e2e-test resourceVersion: "19259" uid: 395bc3c1-2bdc-4dbd-aa3f-82ef64696d25 kind: Event lastTimestamp: "2026-07-08T07:14:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T07:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:14:57Z" name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp.18c03f28f86e8674 namespace: kserve-ci-e2e-test resourceVersion: "19354" uid: ce41f79b-6296-479f-beed-3974a8e01046 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp namespace: kserve-ci-e2e-test resourceVersion: "19259" uid: 395bc3c1-2bdc-4dbd-aa3f-82ef64696d25 kind: Event lastTimestamp: "2026-07-08T07:14:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:14:57Z" name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp.18c03f28f8fd77e6 namespace: kserve-ci-e2e-test resourceVersion: "19355" uid: 366f691c-a997-4f6f-94fe-4a3046f94be5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 18 eventTime: null firstTimestamp: "2026-07-08T07:14:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp namespace: kserve-ci-e2e-test resourceVersion: "19259" uid: 395bc3c1-2bdc-4dbd-aa3f-82ef64696d25 kind: Event lastTimestamp: "2026-07-08T07:17:38Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T07:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:17:38Z" name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp.18c03f292deca6ef namespace: kserve-ci-e2e-test resourceVersion: "20563" uid: e0c062d7-0974-44ea-8be9-3e720edec3a1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:17:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp namespace: kserve-ci-e2e-test resourceVersion: "19259" uid: 395bc3c1-2bdc-4dbd-aa3f-82ef64696d25 kind: Event lastTimestamp: "2026-07-08T07:17:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T07:17:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:17:32Z" name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp.18c03f4d0b22ccc8 namespace: kserve-ci-e2e-test resourceVersion: "20475" uid: 5d2954f6-5e28-4d67-86fe-f0a75d81ab71 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:17:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp namespace: kserve-ci-e2e-test resourceVersion: "19259" uid: 395bc3c1-2bdc-4dbd-aa3f-82ef64696d25 kind: Event lastTimestamp: "2026-07-08T07:17:32Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:17:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:17:32Z" name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp.18c03f4d0b2377c9 namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: ab74c21a-a9fc-457e-9a3e-45301909c2ac reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:17:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp namespace: kserve-ci-e2e-test resourceVersion: "19259" uid: 395bc3c1-2bdc-4dbd-aa3f-82ef64696d25 kind: Event lastTimestamp: "2026-07-08T07:17:39Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T07:17:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:17:39Z" name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp.18c03f4d8075d892 namespace: kserve-ci-e2e-test resourceVersion: "20574" uid: 4683a537-76f9-4306-b4b0-08bc1cb0f876 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:14:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89 namespace: kserve-ci-e2e-test resourceVersion: "19236" uid: 04efef14-1632-4c0e-bce6-c8767a266f68 kind: Event lastTimestamp: "2026-07-08T07:14:52Z" message: 'Created pod: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89-8jqwp' metadata: creationTimestamp: "2026-07-08T07:14:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:14:52Z" name: isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89.18c03f27c5188a36 namespace: kserve-ci-e2e-test resourceVersion: "19258" uid: 5fd4d41e-1d0e-4001-bc3f-9c27757e6289 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:14:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-bba5b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19231" uid: 4614d5ea-a16d-4721-a794-89aafa0c6a1d kind: Event lastTimestamp: "2026-07-08T07:14:52Z" message: Scaled up replica set isvc-sklearn-scale-raw-bba5b-predictor-66899b6f89 from 0 to 1 metadata: creationTimestamp: "2026-07-08T07:14:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:14:52Z" name: isvc-sklearn-scale-raw-bba5b-predictor.18c03f27bed4cf14 namespace: kserve-ci-e2e-test resourceVersion: "19239" uid: cf82aa7f-356b-4dce-b768-439d07a2e8de reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T07:15:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-bba5b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19235" uid: f0132ffc-0271-459e-a665-4f8db0754700 kind: Event lastTimestamp: "2026-07-08T07:15:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:15:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:15:37Z" name: isvc-sklearn-scale-raw-bba5b-predictor.18c03f2b3d94e731 namespace: kserve-ci-e2e-test resourceVersion: "19646" uid: 54abf955-b9fa-4d07-a9dd-f0f26599c52b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T07:15:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-bba5b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19235" uid: f0132ffc-0271-459e-a665-4f8db0754700 kind: Event lastTimestamp: "2026-07-08T07:15:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:15:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:15:37Z" name: isvc-sklearn-scale-raw-bba5b-predictor.18c03f2b3deec008 namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 24e274f2-2398-43bd-be6f-9310d1d689db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-08T07:15:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-bba5b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19429" uid: f0132ffc-0271-459e-a665-4f8db0754700 kind: Event lastTimestamp: "2026-07-08T07:17:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T07:15:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:17:22Z" name: isvc-sklearn-scale-raw-bba5b-predictor.18c03f35bae58018 namespace: kserve-ci-e2e-test resourceVersion: "20380" uid: 8a30ea76-e318-4d4e-8444-68c353cbf050 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-08T07:15:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-bba5b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19429" uid: f0132ffc-0271-459e-a665-4f8db0754700 kind: Event lastTimestamp: "2026-07-08T07:17:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T07:15:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:17:22Z" name: isvc-sklearn-scale-raw-bba5b-predictor.18c03f35bb466bfe namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: d715e790-b9c8-418c-a41d-24a794045f7f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-08T07:14:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-bba5b namespace: kserve-ci-e2e-test resourceVersion: "19216" uid: a228e390-862b-4a31-bae9-c0ca9f7ce999 kind: Event lastTimestamp: "2026-07-08T07:17:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T07:14:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:17:28Z" name: isvc-sklearn-scale-raw-bba5b.18c03f27bf82ac55 namespace: kserve-ci-e2e-test resourceVersion: "20433" uid: 40efc6fc-281b-4062-a52f-7f2457d70fc1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:14:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-bba5b namespace: kserve-ci-e2e-test resourceVersion: "19216" uid: a228e390-862b-4a31-bae9-c0ca9f7ce999 kind: Event lastTimestamp: "2026-07-08T07:14:52Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-bba5b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-bba5b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T07:14:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:14:52Z" name: isvc-sklearn-scale-raw-bba5b.18c03f27c0921061 namespace: kserve-ci-e2e-test resourceVersion: "19283" uid: 31d3100a-b184-432c-9fea-b34785e48952 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:14:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-bba5b namespace: kserve-ci-e2e-test resourceVersion: "19232" uid: a228e390-862b-4a31-bae9-c0ca9f7ce999 kind: Event lastTimestamp: "2026-07-08T07:14:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-bba5b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T07:14:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:14:52Z" name: isvc-sklearn-scale-raw-bba5b.18c03f27c86c4cdf namespace: kserve-ci-e2e-test resourceVersion: "19284" uid: a7a99e97-0e98-4a18-bbc0-11cfb789a16f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:17:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-bba5b namespace: kserve-ci-e2e-test resourceVersion: "20430" uid: a228e390-862b-4a31-bae9-c0ca9f7ce999 kind: Event lastTimestamp: "2026-07-08T07:17:28Z" message: InferenceService [isvc-sklearn-scale-raw-bba5b] is Ready metadata: creationTimestamp: "2026-07-08T07:17:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:17:28Z" name: isvc-sklearn-scale-raw-bba5b.18c03f4c2eb26646 namespace: kserve-ci-e2e-test resourceVersion: "20431" uid: 99976812-1290-485d-a121-bae352994527 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T07:09:16.551614Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: 24ee847f-88cd-45b6-854f-a0f2c419f0b7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch to ip-10-0-135-211.ec2.internal metadata: creationTimestamp: "2026-07-08T07:09:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T07:09:16Z" name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch.18c03ed98e1a386d namespace: kserve-ci-e2e-test resourceVersion: "16202" uid: 57f18f19-e990-4d5c-af1b-fc5b1ec8dc7d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fb64cb696-6mkqx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 24ee847f-88cd-45b6-854f-a0f2c419f0b7 kind: Event lastTimestamp: "2026-07-08T07:09:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-a4509-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T07:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:16Z" name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch.18c03ed99e27db10 namespace: kserve-ci-e2e-test resourceVersion: "16220" uid: d2b6d0ad-2577-4dc1-9af8-fb2c33b0be29 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:17Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch namespace: kserve-ci-e2e-test resourceVersion: "16207" uid: 24ee847f-88cd-45b6-854f-a0f2c419f0b7 kind: Event lastTimestamp: "2026-07-08T07:09:17Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T07:09:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T07:09:17Z" name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch.18c03ed9cbb8be24 namespace: kserve-ci-e2e-test resourceVersion: "16238" uid: f2c2c958-56b9-4586-820b-96f9180629ad reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 24ee847f-88cd-45b6-854f-a0f2c419f0b7 kind: Event lastTimestamp: "2026-07-08T07:09:17Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773" already present on machine metadata: creationTimestamp: "2026-07-08T07:09:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:17Z" name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch.18c03ed9cd267a01 namespace: kserve-ci-e2e-test resourceVersion: "16240" uid: 0de8aed3-3d35-4a06-856a-35077cfd9125 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 24ee847f-88cd-45b6-854f-a0f2c419f0b7 kind: Event lastTimestamp: "2026-07-08T07:09:17Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T07:09:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:17Z" name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch.18c03ed9d05fe29d namespace: kserve-ci-e2e-test resourceVersion: "16243" uid: b4d75999-a0d0-4580-95fc-42d26942aa44 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 24ee847f-88cd-45b6-854f-a0f2c419f0b7 kind: Event lastTimestamp: "2026-07-08T07:09:17Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T07:09:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:17Z" name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch.18c03ed9d0f62ebf namespace: kserve-ci-e2e-test resourceVersion: "16244" uid: 29970452-b267-4b64-9308-df9cd9822ed9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 24ee847f-88cd-45b6-854f-a0f2c419f0b7 kind: Event lastTimestamp: "2026-07-08T07:09:22Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-08T07:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:22Z" name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch.18c03edad6d2913b namespace: kserve-ci-e2e-test resourceVersion: "16292" uid: a04dacfd-b235-4f74-9824-b5bcff92f319 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 24ee847f-88cd-45b6-854f-a0f2c419f0b7 kind: Event lastTimestamp: "2026-07-08T07:09:42Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 20.096s (20.096s including waiting). Image size: 1305547509 bytes.' metadata: creationTimestamp: "2026-07-08T07:09:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:42Z" name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch.18c03edf84acd644 namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: b4b012c4-8607-43b7-9893-086998cad867 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 24ee847f-88cd-45b6-854f-a0f2c419f0b7 kind: Event lastTimestamp: "2026-07-08T07:09:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T07:09:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:42Z" name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch.18c03edf88a2018c namespace: kserve-ci-e2e-test resourceVersion: "16461" uid: afebab19-578a-4214-93b9-4b38b120f9e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 24ee847f-88cd-45b6-854f-a0f2c419f0b7 kind: Event lastTimestamp: "2026-07-08T07:09:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T07:09:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:42Z" name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch.18c03edf891659c9 namespace: kserve-ci-e2e-test resourceVersion: "16462" uid: 791be72c-1024-435f-a3a1-38da0309b1bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 24ee847f-88cd-45b6-854f-a0f2c419f0b7 kind: Event lastTimestamp: "2026-07-08T07:09:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T07:09:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:42Z" name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch.18c03edf891faec8 namespace: kserve-ci-e2e-test resourceVersion: "16463" uid: 12d3f557-009e-4a26-8c96-1d89204e59a5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 24ee847f-88cd-45b6-854f-a0f2c419f0b7 kind: Event lastTimestamp: "2026-07-08T07:09:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T07:09:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:42Z" name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch.18c03edf8cba4aa1 namespace: kserve-ci-e2e-test resourceVersion: "16464" uid: 29bffca6-561d-4883-b72f-41ec4ad84420 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 24ee847f-88cd-45b6-854f-a0f2c419f0b7 kind: Event lastTimestamp: "2026-07-08T07:09:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:09:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:09:42Z" name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch.18c03edf8d2e1f5c namespace: kserve-ci-e2e-test resourceVersion: "16465" uid: 10a7839e-119f-4d45-acfc-8aa16ae69047 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-08T07:09:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 24ee847f-88cd-45b6-854f-a0f2c419f0b7 kind: Event lastTimestamp: "2026-07-08T07:11:09Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T07:09:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:09Z" name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch.18c03edfc103bf39 namespace: kserve-ci-e2e-test resourceVersion: "17310" uid: 8a29ce6e-754d-4fa9-b0b0-0ed8cd2c0de2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 24ee847f-88cd-45b6-854f-a0f2c419f0b7 kind: Event lastTimestamp: "2026-07-08T07:11:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T07:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:06Z" name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch.18c03ef33eec770f namespace: kserve-ci-e2e-test resourceVersion: "17256" uid: 9fdca7e7-f8bc-4994-b54b-dd8a82d34017 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 24ee847f-88cd-45b6-854f-a0f2c419f0b7 kind: Event lastTimestamp: "2026-07-08T07:11:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:06Z" name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch.18c03ef33eed00d5 namespace: kserve-ci-e2e-test resourceVersion: "17258" uid: 878d4a1e-2761-46fa-b6fa-82399b3c9c58 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 24ee847f-88cd-45b6-854f-a0f2c419f0b7 kind: Event lastTimestamp: "2026-07-08T07:11:09Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp 10.133.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T07:11:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:09Z" name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch.18c03ef3c73f3722 namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: 68f4e142-9c07-426d-ad98-64c321751052 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf namespace: kserve-ci-e2e-test resourceVersion: "16185" uid: c2aa73c6-4de0-42c1-a60c-120370a281bb kind: Event lastTimestamp: "2026-07-08T07:09:16Z" message: 'Created pod: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf-2dxch' metadata: creationTimestamp: "2026-07-08T07:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:09:16Z" name: isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf.18c03ed98da9c634 namespace: kserve-ci-e2e-test resourceVersion: "16199" uid: fb5f5f3a-bfa2-469a-b7c1-0b8acac51714 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-a4509-predictor namespace: kserve-ci-e2e-test resourceVersion: "16180" uid: 7ea4a15a-1b36-4a1b-a5ae-e57f03c684b9 kind: Event lastTimestamp: "2026-07-08T07:09:16Z" message: Scaled up replica set isvc-xgboost-graph-raw-a4509-predictor-67c9b5b5bf from 0 to 1 metadata: creationTimestamp: "2026-07-08T07:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:09:16Z" name: isvc-xgboost-graph-raw-a4509-predictor.18c03ed98ad98340 namespace: kserve-ci-e2e-test resourceVersion: "16188" uid: 4db903bf-bee1-4bde-86ea-95db1ee8a11a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T07:09:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-a4509-predictor namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: d5bc6ada-ec42-4afb-abee-45f654b4e9d6 kind: Event lastTimestamp: "2026-07-08T07:10:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:10:16Z" name: isvc-xgboost-graph-raw-a4509-predictor.18c03edd095747cb namespace: kserve-ci-e2e-test resourceVersion: "16722" uid: 93a4b849-5ea1-4515-afe6-c4a1d056b930 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T07:09:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-a4509-predictor namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: d5bc6ada-ec42-4afb-abee-45f654b4e9d6 kind: Event lastTimestamp: "2026-07-08T07:10:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:10:16Z" name: isvc-xgboost-graph-raw-a4509-predictor.18c03edd09caf616 namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: a3a1d9d3-3842-450c-82f1-d9565b8119ce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T07:10:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-a4509-predictor namespace: kserve-ci-e2e-test resourceVersion: "16379" uid: d5bc6ada-ec42-4afb-abee-45f654b4e9d6 kind: Event lastTimestamp: "2026-07-08T07:11:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T07:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:11:01Z" name: isvc-xgboost-graph-raw-a4509-predictor.18c03eeb04952285 namespace: kserve-ci-e2e-test resourceVersion: "17108" uid: fa0240e3-7365-49f5-9487-84d1d86f226a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T07:10:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-a4509-predictor namespace: kserve-ci-e2e-test resourceVersion: "16379" uid: d5bc6ada-ec42-4afb-abee-45f654b4e9d6 kind: Event lastTimestamp: "2026-07-08T07:11:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T07:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:11:01Z" name: isvc-xgboost-graph-raw-a4509-predictor.18c03eeb04f867b1 namespace: kserve-ci-e2e-test resourceVersion: "17109" uid: 9af3b485-7ec8-43d2-8bdd-1e3a13f007f7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T07:09:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-a4509 namespace: kserve-ci-e2e-test resourceVersion: "16147" uid: 3d823608-ac78-4f66-853c-103a111a80b7 kind: Event lastTimestamp: "2026-07-08T07:10:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T07:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:10:39Z" name: isvc-xgboost-graph-raw-a4509.18c03ed98b11bbfb namespace: kserve-ci-e2e-test resourceVersion: "16910" uid: 719ca950-971d-40c9-b2d6-8035600614dd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:09:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-a4509 namespace: kserve-ci-e2e-test resourceVersion: "16147" uid: 3d823608-ac78-4f66-853c-103a111a80b7 kind: Event lastTimestamp: "2026-07-08T07:09:16Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-a4509": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-a4509": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T07:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:09:16Z" name: isvc-xgboost-graph-raw-a4509.18c03ed98b75ca50 namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: a10756f5-1158-4ee7-8cee-3b9c13dc606a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:10:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-a4509 namespace: kserve-ci-e2e-test resourceVersion: "16907" uid: 3d823608-ac78-4f66-853c-103a111a80b7 kind: Event lastTimestamp: "2026-07-08T07:10:39Z" message: InferenceService [isvc-xgboost-graph-raw-a4509] is Ready metadata: creationTimestamp: "2026-07-08T07:10:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:10:39Z" name: isvc-xgboost-graph-raw-a4509.18c03eecd017b3c3 namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: c78383a2-e2d0-4d9a-9d56-3154e5ccf7f9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T07:11:06.853089Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v namespace: kserve-ci-e2e-test resourceVersion: "17228" uid: 59ade4e3-fb68-4453-95ec-6f8ac3f136a8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v to ip-10-0-135-211.ec2.internal metadata: creationTimestamp: "2026-07-08T07:11:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T07:11:06Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v.18c03ef33c95312b namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: 264f4b01-189a-4920-98c6-7f49f59c5a06 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fb64cb696-6mkqx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v namespace: kserve-ci-e2e-test resourceVersion: "17231" uid: 59ade4e3-fb68-4453-95ec-6f8ac3f136a8 kind: Event lastTimestamp: "2026-07-08T07:11:07Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-hpa-aea5a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T07:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:07Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v.18c03ef34bf8dc19 namespace: kserve-ci-e2e-test resourceVersion: "17267" uid: 635be632-16b4-4784-a90e-8a9518b0f8d3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v namespace: kserve-ci-e2e-test resourceVersion: "17253" uid: 59ade4e3-fb68-4453-95ec-6f8ac3f136a8 kind: Event lastTimestamp: "2026-07-08T07:11:07Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T07:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T07:11:07Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v.18c03ef379e990d4 namespace: kserve-ci-e2e-test resourceVersion: "17287" uid: 9d438ee3-53c4-495b-8216-72341f9fc567 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v namespace: kserve-ci-e2e-test resourceVersion: "17231" uid: 59ade4e3-fb68-4453-95ec-6f8ac3f136a8 kind: Event lastTimestamp: "2026-07-08T07:11:07Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773" already present on machine metadata: creationTimestamp: "2026-07-08T07:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:07Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v.18c03ef37b660eea namespace: kserve-ci-e2e-test resourceVersion: "17289" uid: f0b9f4b5-fee8-44f4-8d70-70fea4c9ef85 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v namespace: kserve-ci-e2e-test resourceVersion: "17231" uid: 59ade4e3-fb68-4453-95ec-6f8ac3f136a8 kind: Event lastTimestamp: "2026-07-08T07:11:07Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T07:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:07Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v.18c03ef37ebe4c10 namespace: kserve-ci-e2e-test resourceVersion: "17291" uid: 502d7639-615b-47df-8445-55232db00264 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v namespace: kserve-ci-e2e-test resourceVersion: "17231" uid: 59ade4e3-fb68-4453-95ec-6f8ac3f136a8 kind: Event lastTimestamp: "2026-07-08T07:11:07Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T07:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:07Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v.18c03ef37f2e6d2f namespace: kserve-ci-e2e-test resourceVersion: "17292" uid: 435de6f8-bd84-43cb-ac8b-d97605c894ef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v namespace: kserve-ci-e2e-test resourceVersion: "17231" uid: 59ade4e3-fb68-4453-95ec-6f8ac3f136a8 kind: Event lastTimestamp: "2026-07-08T07:11:12Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-07-08T07:11:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:12Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v.18c03ef490c55d4d namespace: kserve-ci-e2e-test resourceVersion: "17348" uid: ceedbf00-7534-4e7c-8d87-013faa33a540 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v namespace: kserve-ci-e2e-test resourceVersion: "17231" uid: 59ade4e3-fb68-4453-95ec-6f8ac3f136a8 kind: Event lastTimestamp: "2026-07-08T07:11:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T07:11:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:12Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v.18c03ef493bcd819 namespace: kserve-ci-e2e-test resourceVersion: "17352" uid: 2eb0d4ad-79bf-4872-8bac-07de6b283623 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v namespace: kserve-ci-e2e-test resourceVersion: "17231" uid: 59ade4e3-fb68-4453-95ec-6f8ac3f136a8 kind: Event lastTimestamp: "2026-07-08T07:11:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T07:11:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:12Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v.18c03ef49433ff5d namespace: kserve-ci-e2e-test resourceVersion: "17354" uid: e69eb554-0c28-43ac-b816-74a48c4f5af5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v namespace: kserve-ci-e2e-test resourceVersion: "17231" uid: 59ade4e3-fb68-4453-95ec-6f8ac3f136a8 kind: Event lastTimestamp: "2026-07-08T07:11:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T07:11:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:12Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v.18c03ef4943db2c3 namespace: kserve-ci-e2e-test resourceVersion: "17356" uid: 55dc03b7-5465-471e-9b3e-0ac72446163f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v namespace: kserve-ci-e2e-test resourceVersion: "17231" uid: 59ade4e3-fb68-4453-95ec-6f8ac3f136a8 kind: Event lastTimestamp: "2026-07-08T07:11:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T07:11:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:12Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v.18c03ef4975665de namespace: kserve-ci-e2e-test resourceVersion: "17360" uid: d68758cb-9301-4884-9268-19e177008402 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v namespace: kserve-ci-e2e-test resourceVersion: "17231" uid: 59ade4e3-fb68-4453-95ec-6f8ac3f136a8 kind: Event lastTimestamp: "2026-07-08T07:11:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:11:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:12Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v.18c03ef497c9ea77 namespace: kserve-ci-e2e-test resourceVersion: "17361" uid: 639ddc5b-bf99-4aa8-babc-517b4108035f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-08T07:11:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v namespace: kserve-ci-e2e-test resourceVersion: "17231" uid: 59ade4e3-fb68-4453-95ec-6f8ac3f136a8 kind: Event lastTimestamp: "2026-07-08T07:12:59Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T07:11:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:12:59Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v.18c03ef4ccf47c98 namespace: kserve-ci-e2e-test resourceVersion: "18305" uid: ad720b0b-9103-4f34-9f79-60709df89637 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v namespace: kserve-ci-e2e-test resourceVersion: "17231" uid: 59ade4e3-fb68-4453-95ec-6f8ac3f136a8 kind: Event lastTimestamp: "2026-07-08T07:12:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T07:12:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:12:57Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v.18c03f0ced6e8276 namespace: kserve-ci-e2e-test resourceVersion: "18263" uid: 37bfae96-fa20-44bc-85d3-f3ac1fe412e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v namespace: kserve-ci-e2e-test resourceVersion: "17231" uid: 59ade4e3-fb68-4453-95ec-6f8ac3f136a8 kind: Event lastTimestamp: "2026-07-08T07:12:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:12:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:12:57Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v.18c03f0ced6f0409 namespace: kserve-ci-e2e-test resourceVersion: "18265" uid: ab5a874f-0022-4c75-ada1-06c7fae92fd9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v namespace: kserve-ci-e2e-test resourceVersion: "17231" uid: 59ade4e3-fb68-4453-95ec-6f8ac3f136a8 kind: Event lastTimestamp: "2026-07-08T07:12:59Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T07:12:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:12:59Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v.18c03f0d7b5679c1 namespace: kserve-ci-e2e-test resourceVersion: "18304" uid: 1780fdf4-ee8e-4de0-8672-2f9c41b20225 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df namespace: kserve-ci-e2e-test resourceVersion: "17221" uid: c0ef7f49-622e-4709-bc5c-7624dd74c9cf kind: Event lastTimestamp: "2026-07-08T07:11:06Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df-w926v' metadata: creationTimestamp: "2026-07-08T07:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:11:06Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df.18c03ef33c2127db namespace: kserve-ci-e2e-test resourceVersion: "17230" uid: 0b51f529-6b6b-4541-b1e8-e0b36cbd0afa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-aea5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17219" uid: 148dbe21-b0f5-40cf-a96e-7638848fb761 kind: Event lastTimestamp: "2026-07-08T07:11:06Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-aea5a-predictor-695b4769df from 0 to 1 metadata: creationTimestamp: "2026-07-08T07:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:11:06Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor.18c03ef339705e60 namespace: kserve-ci-e2e-test resourceVersion: "17223" uid: 1aae26b4-e0ba-4749-b105-8481cc54f671 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T07:11:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-aea5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17220" uid: 09c51a5e-130a-4bc2-9fcc-62256ffa74d7 kind: Event lastTimestamp: "2026-07-08T07:11:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:11:51Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor.18c03ef6b7e3a154 namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 12429eef-85e0-44c3-8bb7-04714e7aca81 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T07:11:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-aea5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17220" uid: 09c51a5e-130a-4bc2-9fcc-62256ffa74d7 kind: Event lastTimestamp: "2026-07-08T07:11:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:11:51Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor.18c03ef6b8446c7f namespace: kserve-ci-e2e-test resourceVersion: "17664" uid: f8a732e1-eda5-4cb3-a253-d079bd894819 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T07:12:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-aea5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17440" uid: 09c51a5e-130a-4bc2-9fcc-62256ffa74d7 kind: Event lastTimestamp: "2026-07-08T07:12:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T07:12:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:12:36Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor.18c03f0134343a56 namespace: kserve-ci-e2e-test resourceVersion: "17999" uid: 7da1821c-7434-458d-af86-e5e3a264d3b0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T07:12:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-aea5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17440" uid: 09c51a5e-130a-4bc2-9fcc-62256ffa74d7 kind: Event lastTimestamp: "2026-07-08T07:12:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T07:12:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:12:36Z" name: isvc-xgboost-graph-raw-hpa-aea5a-predictor.18c03f013493045e namespace: kserve-ci-e2e-test resourceVersion: "18000" uid: ad9bad8b-9983-447e-91d4-05698069503e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T07:11:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-aea5a namespace: kserve-ci-e2e-test resourceVersion: "17200" uid: e0943f63-15f7-4d22-af42-5076e77e2851 kind: Event lastTimestamp: "2026-07-08T07:12:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T07:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:12:09Z" name: isvc-xgboost-graph-raw-hpa-aea5a.18c03ef3397c76aa namespace: kserve-ci-e2e-test resourceVersion: "17799" uid: 8df773ba-8985-4f93-adc8-ec89ddd136b2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-aea5a namespace: kserve-ci-e2e-test resourceVersion: "17200" uid: e0943f63-15f7-4d22-af42-5076e77e2851 kind: Event lastTimestamp: "2026-07-08T07:11:06Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-aea5a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-aea5a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T07:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:11:06Z" name: isvc-xgboost-graph-raw-hpa-aea5a.18c03ef339eca985 namespace: kserve-ci-e2e-test resourceVersion: "17226" uid: 7df378da-8fa3-4587-bbeb-6532a30e90ab reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-aea5a namespace: kserve-ci-e2e-test resourceVersion: "17797" uid: e0943f63-15f7-4d22-af42-5076e77e2851 kind: Event lastTimestamp: "2026-07-08T07:12:09Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-aea5a] is Ready metadata: creationTimestamp: "2026-07-08T07:12:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:12:09Z" name: isvc-xgboost-graph-raw-hpa-aea5a.18c03f01dc94c9d0 namespace: kserve-ci-e2e-test resourceVersion: "17798" uid: 473c0ab3-eedf-4fe3-a207-3fb92c1793e7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T07:12:57.051082Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk namespace: kserve-ci-e2e-test resourceVersion: "18216" uid: b4e94490-b45d-4292-baf1-0aa81f223a21 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-53e83-predictor-78879675f8-nsgwk to ip-10-0-135-211.ec2.internal metadata: creationTimestamp: "2026-07-08T07:12:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T07:12:57Z" name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk.18c03f0ce4e51239 namespace: kserve-ci-e2e-test resourceVersion: "18224" uid: aeb4d425-ca9b-4ab3-aff7-f3514a9be39a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fb64cb696-6mkqx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:57Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk namespace: kserve-ci-e2e-test resourceVersion: "18237" uid: b4e94490-b45d-4292-baf1-0aa81f223a21 kind: Event lastTimestamp: "2026-07-08T07:12:57Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T07:12:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T07:12:57Z" name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk.18c03f0cfea69799 namespace: kserve-ci-e2e-test resourceVersion: "18276" uid: 5c27fa04-6b97-4108-b2ea-da1e1254df1c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk namespace: kserve-ci-e2e-test resourceVersion: "18218" uid: b4e94490-b45d-4292-baf1-0aa81f223a21 kind: Event lastTimestamp: "2026-07-08T07:12:57Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-08T07:12:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:12:57Z" name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk.18c03f0cffea81d2 namespace: kserve-ci-e2e-test resourceVersion: "18278" uid: e8f23287-8e26-420b-9a0f-063f9681b65d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk namespace: kserve-ci-e2e-test resourceVersion: "18218" uid: b4e94490-b45d-4292-baf1-0aa81f223a21 kind: Event lastTimestamp: "2026-07-08T07:12:58Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.287s (1.287s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-08T07:12:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:12:58Z" name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk.18c03f0d4cae7940 namespace: kserve-ci-e2e-test resourceVersion: "18289" uid: 3557f3a5-cf71-48c6-b605-fd83964991f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk namespace: kserve-ci-e2e-test resourceVersion: "18218" uid: b4e94490-b45d-4292-baf1-0aa81f223a21 kind: Event lastTimestamp: "2026-07-08T07:12:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T07:12:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:12:58Z" name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk.18c03f0d500e99e1 namespace: kserve-ci-e2e-test resourceVersion: "18291" uid: 0002156e-c6a4-41af-a614-902defb45e61 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk namespace: kserve-ci-e2e-test resourceVersion: "18218" uid: b4e94490-b45d-4292-baf1-0aa81f223a21 kind: Event lastTimestamp: "2026-07-08T07:12:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T07:12:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:12:58Z" name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk.18c03f0d50829913 namespace: kserve-ci-e2e-test resourceVersion: "18292" uid: df2dd9fa-5e58-4dff-b2ec-2e7ea419ac28 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk namespace: kserve-ci-e2e-test resourceVersion: "18218" uid: b4e94490-b45d-4292-baf1-0aa81f223a21 kind: Event lastTimestamp: "2026-07-08T07:12:58Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T07:12:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:12:58Z" name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk.18c03f0d508b6e40 namespace: kserve-ci-e2e-test resourceVersion: "18293" uid: cedee21e-f8e5-492a-9167-8106c8537275 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk namespace: kserve-ci-e2e-test resourceVersion: "18218" uid: b4e94490-b45d-4292-baf1-0aa81f223a21 kind: Event lastTimestamp: "2026-07-08T07:12:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T07:12:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:12:58Z" name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk.18c03f0d53604b01 namespace: kserve-ci-e2e-test resourceVersion: "18294" uid: 303e8094-303a-499d-8ec9-56d040bdf239 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk namespace: kserve-ci-e2e-test resourceVersion: "18218" uid: b4e94490-b45d-4292-baf1-0aa81f223a21 kind: Event lastTimestamp: "2026-07-08T07:12:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:12:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:12:58Z" name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk.18c03f0d53d53dd8 namespace: kserve-ci-e2e-test resourceVersion: "18295" uid: c743d333-8c1f-4ca6-a57d-314e5c0aa22c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:14:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk namespace: kserve-ci-e2e-test resourceVersion: "18218" uid: b4e94490-b45d-4292-baf1-0aa81f223a21 kind: Event lastTimestamp: "2026-07-08T07:14:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T07:14:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:14:52Z" name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk.18c03f27c7b23e62 namespace: kserve-ci-e2e-test resourceVersion: "19279" uid: 2df73e47-4876-4e37-8444-eb5c6e8d8acd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:14:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk namespace: kserve-ci-e2e-test resourceVersion: "18218" uid: b4e94490-b45d-4292-baf1-0aa81f223a21 kind: Event lastTimestamp: "2026-07-08T07:14:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:14:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:14:52Z" name: message-dumper-raw-53e83-predictor-78879675f8-nsgwk.18c03f27c7b36604 namespace: kserve-ci-e2e-test resourceVersion: "19281" uid: 71a8c953-de12-4921-b714-7a37e1d27de0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-53e83-predictor-78879675f8 namespace: kserve-ci-e2e-test resourceVersion: "18207" uid: 19895bd9-1c5a-4d14-8bda-778524200785 kind: Event lastTimestamp: "2026-07-08T07:12:57Z" message: 'Created pod: message-dumper-raw-53e83-predictor-78879675f8-nsgwk' metadata: creationTimestamp: "2026-07-08T07:12:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:12:57Z" name: message-dumper-raw-53e83-predictor-78879675f8.18c03f0ce47db1c6 namespace: kserve-ci-e2e-test resourceVersion: "18217" uid: 2d51764b-5611-4077-abee-5e06dcd13574 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-53e83-predictor namespace: kserve-ci-e2e-test resourceVersion: "18205" uid: fcc7bf1e-81f0-4572-9ae8-101e8ec8a317 kind: Event lastTimestamp: "2026-07-08T07:12:56Z" message: Scaled up replica set message-dumper-raw-53e83-predictor-78879675f8 from 0 to 1 metadata: creationTimestamp: "2026-07-08T07:12:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:12:56Z" name: message-dumper-raw-53e83-predictor.18c03f0ce154258f namespace: kserve-ci-e2e-test resourceVersion: "18208" uid: b4d3c09c-d7f0-4537-933b-649f8342ab2f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:13:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-53e83-predictor namespace: kserve-ci-e2e-test resourceVersion: "18206" uid: cadb5ad0-12c3-4a7d-bb9a-209b3fd8db95 kind: Event lastTimestamp: "2026-07-08T07:13:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:13:27Z" name: message-dumper-raw-53e83-predictor.18c03f105fdc78ac namespace: kserve-ci-e2e-test resourceVersion: "18594" uid: 4f70d7fc-25ea-415e-9f9d-f899d6e25d41 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:13:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-53e83-predictor namespace: kserve-ci-e2e-test resourceVersion: "18206" uid: cadb5ad0-12c3-4a7d-bb9a-209b3fd8db95 kind: Event lastTimestamp: "2026-07-08T07:13:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:13:27Z" name: message-dumper-raw-53e83-predictor.18c03f10603a25cf namespace: kserve-ci-e2e-test resourceVersion: "18595" uid: 174292f9-03dd-47d6-b935-8a5e56002a8f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-08T07:12:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-53e83 namespace: kserve-ci-e2e-test resourceVersion: "18197" uid: a2dde2b4-9956-4a22-8f4f-9368aa89993a kind: Event lastTimestamp: "2026-07-08T07:13:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T07:12:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:13:07Z" name: message-dumper-raw-53e83.18c03f0ce15a6007 namespace: kserve-ci-e2e-test resourceVersion: "18375" uid: 115fc249-3e75-487c-adcb-6fa975471cbc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:12:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-53e83 namespace: kserve-ci-e2e-test resourceVersion: "18197" uid: a2dde2b4-9956-4a22-8f4f-9368aa89993a kind: Event lastTimestamp: "2026-07-08T07:12:57Z" message: 'Failed to update status for InferenceService "message-dumper-raw-53e83": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-53e83": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T07:12:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:12:57Z" name: message-dumper-raw-53e83.18c03f0ce1d7e895 namespace: kserve-ci-e2e-test resourceVersion: "18268" uid: da6a5915-ac2b-4da3-b5bf-27fe18f52085 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-53e83 namespace: kserve-ci-e2e-test resourceVersion: "18236" uid: a2dde2b4-9956-4a22-8f4f-9368aa89993a kind: Event lastTimestamp: "2026-07-08T07:12:57Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-53e83": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T07:12:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:12:57Z" name: message-dumper-raw-53e83.18c03f0cefe3a7b6 namespace: kserve-ci-e2e-test resourceVersion: "18269" uid: 32b54f24-f036-49dd-a6f7-aa1e7e5a507a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:13:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-53e83 namespace: kserve-ci-e2e-test resourceVersion: "18373" uid: a2dde2b4-9956-4a22-8f4f-9368aa89993a kind: Event lastTimestamp: "2026-07-08T07:13:07Z" message: InferenceService [message-dumper-raw-53e83] is Ready metadata: creationTimestamp: "2026-07-08T07:13:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:13:07Z" name: message-dumper-raw-53e83.18c03f0f3bc3d544 namespace: kserve-ci-e2e-test resourceVersion: "18374" uid: ae66fb2e-971d-45a2-b80f-19fce47171ac reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T07:10:56.517145Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-a4509-69b99bcd9d-ttw4x namespace: kserve-ci-e2e-test resourceVersion: "17039" uid: e93cd997-3d55-41e1-844d-3511aaca9cfa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-a4509-69b99bcd9d-ttw4x to ip-10-0-138-98.ec2.internal metadata: creationTimestamp: "2026-07-08T07:10:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T07:10:56Z" name: model-chainer-raw-a4509-69b99bcd9d-ttw4x.18c03ef0d48316f8 namespace: kserve-ci-e2e-test resourceVersion: "17045" uid: c6107a89-ef07-4949-b586-93972d2245f9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fb64cb696-6mkqx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:10:56Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-a4509-69b99bcd9d-ttw4x namespace: kserve-ci-e2e-test resourceVersion: "17048" uid: e93cd997-3d55-41e1-844d-3511aaca9cfa kind: Event lastTimestamp: "2026-07-08T07:10:56Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T07:10:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T07:10:56Z" name: model-chainer-raw-a4509-69b99bcd9d-ttw4x.18c03ef0ee438fea namespace: kserve-ci-e2e-test resourceVersion: "17058" uid: 584a9056-acac-454a-af4f-16c34f9b52fb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:10:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-a4509} kind: Pod name: model-chainer-raw-a4509-69b99bcd9d-ttw4x namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: e93cd997-3d55-41e1-844d-3511aaca9cfa kind: Event lastTimestamp: "2026-07-08T07:10:56Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:343c9e455f840436fd086519b55f7daa1173afbafc1b91e08f3c6de9e52e6594" metadata: creationTimestamp: "2026-07-08T07:10:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:10:56Z" name: model-chainer-raw-a4509-69b99bcd9d-ttw4x.18c03ef0ef98d9f1 namespace: kserve-ci-e2e-test resourceVersion: "17060" uid: f1716f49-6c7f-456a-9e24-4b18549cc951 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-98.ec2.internal source: component: kubelet host: ip-10-0-138-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:10:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-a4509} kind: Pod name: model-chainer-raw-a4509-69b99bcd9d-ttw4x namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: e93cd997-3d55-41e1-844d-3511aaca9cfa kind: Event lastTimestamp: "2026-07-08T07:10:59Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:343c9e455f840436fd086519b55f7daa1173afbafc1b91e08f3c6de9e52e6594" in 2.301s (2.301s including waiting). Image size: 218072678 bytes.' metadata: creationTimestamp: "2026-07-08T07:10:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:10:59Z" name: model-chainer-raw-a4509-69b99bcd9d-ttw4x.18c03ef178c6e4ec namespace: kserve-ci-e2e-test resourceVersion: "17082" uid: 7b627124-5669-4d5c-a00c-d541bcba11b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-98.ec2.internal source: component: kubelet host: ip-10-0-138-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:10:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-a4509} kind: Pod name: model-chainer-raw-a4509-69b99bcd9d-ttw4x namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: e93cd997-3d55-41e1-844d-3511aaca9cfa kind: Event lastTimestamp: "2026-07-08T07:10:59Z" message: 'Created container: model-chainer-raw-a4509' metadata: creationTimestamp: "2026-07-08T07:10:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:10:59Z" name: model-chainer-raw-a4509-69b99bcd9d-ttw4x.18c03ef17c0702a3 namespace: kserve-ci-e2e-test resourceVersion: "17083" uid: 65174e36-ddb9-4a38-befe-c036648caa48 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-98.ec2.internal source: component: kubelet host: ip-10-0-138-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:10:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-a4509} kind: Pod name: model-chainer-raw-a4509-69b99bcd9d-ttw4x namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: e93cd997-3d55-41e1-844d-3511aaca9cfa kind: Event lastTimestamp: "2026-07-08T07:10:59Z" message: Started container model-chainer-raw-a4509 metadata: creationTimestamp: "2026-07-08T07:10:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:10:59Z" name: model-chainer-raw-a4509-69b99bcd9d-ttw4x.18c03ef17c989e38 namespace: kserve-ci-e2e-test resourceVersion: "17084" uid: e6c37218-202d-49d4-80d2-e4969e8cdfd1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-98.ec2.internal source: component: kubelet host: ip-10-0-138-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-a4509} kind: Pod name: model-chainer-raw-a4509-69b99bcd9d-ttw4x namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: e93cd997-3d55-41e1-844d-3511aaca9cfa kind: Event lastTimestamp: "2026-07-08T07:11:06Z" message: Stopping container model-chainer-raw-a4509 metadata: creationTimestamp: "2026-07-08T07:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:06Z" name: model-chainer-raw-a4509-69b99bcd9d-ttw4x.18c03ef32a5c7b50 namespace: kserve-ci-e2e-test resourceVersion: "17157" uid: 7a96c45f-d6cc-435d-8f6a-36189e01bf14 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-98.ec2.internal source: component: kubelet host: ip-10-0-138-98.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T07:11:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-a4509} kind: Pod name: model-chainer-raw-a4509-69b99bcd9d-ttw4x namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: e93cd997-3d55-41e1-844d-3511aaca9cfa kind: Event lastTimestamp: "2026-07-08T07:11:35Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T07:11:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:11:35Z" name: model-chainer-raw-a4509-69b99bcd9d-ttw4x.18c03ef43076182a namespace: kserve-ci-e2e-test resourceVersion: "17538" uid: d2230fcd-934c-43c0-956c-561200d35df1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-98.ec2.internal source: component: kubelet host: ip-10-0-138-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:10:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-a4509-69b99bcd9d namespace: kserve-ci-e2e-test resourceVersion: "17036" uid: b7454203-7e8a-4ca6-80b6-e367044a9b2c kind: Event lastTimestamp: "2026-07-08T07:10:56Z" message: 'Created pod: model-chainer-raw-a4509-69b99bcd9d-ttw4x' metadata: creationTimestamp: "2026-07-08T07:10:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:10:56Z" name: model-chainer-raw-a4509-69b99bcd9d.18c03ef0d3bf1513 namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: 82fe2f4c-1f62-4161-964a-79967ced4916 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:10:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-a4509 namespace: kserve-ci-e2e-test resourceVersion: "17034" uid: 8478b2ef-d91f-46f7-93c1-0d972cb0f6d2 kind: Event lastTimestamp: "2026-07-08T07:10:56Z" message: Scaled up replica set model-chainer-raw-a4509-69b99bcd9d from 0 to 1 metadata: creationTimestamp: "2026-07-08T07:10:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:10:56Z" name: model-chainer-raw-a4509.18c03ef0d0f984de namespace: kserve-ci-e2e-test resourceVersion: "17038" uid: 730f24f2-b699-4310-8ed5-da1d23667749 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:10:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-a4509 namespace: kserve-ci-e2e-test resourceVersion: "17050" uid: cae29239-a5bb-4bf8-9c28-dd3fa5b6392e kind: Event lastTimestamp: "2026-07-08T07:10:56Z" message: InferenceGraph [model-chainer-raw-a4509] is Ready metadata: creationTimestamp: "2026-07-08T07:10:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:10:56Z" name: model-chainer-raw-a4509.18c03ef0d928057c namespace: kserve-ci-e2e-test resourceVersion: "17052" uid: 98140392-11ec-4b93-9bb2-63d870fd6798 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T07:12:46.791971Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-aea5a-66fdcfdbb4-j89r4 namespace: kserve-ci-e2e-test resourceVersion: "18082" uid: 398fbe6e-9745-42da-a375-8b87fb80c26d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-aea5a-66fdcfdbb4-j89r4 to ip-10-0-138-98.ec2.internal metadata: creationTimestamp: "2026-07-08T07:12:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T07:12:46Z" name: model-chainer-raw-hpa-aea5a-66fdcfdbb4-j89r4.18c03f0a81678339 namespace: kserve-ci-e2e-test resourceVersion: "18087" uid: 69c716e6-56a2-48cf-abe6-bc5c7a133765 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fb64cb696-6mkqx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:47Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-aea5a-66fdcfdbb4-j89r4 namespace: kserve-ci-e2e-test resourceVersion: "18091" uid: 398fbe6e-9745-42da-a375-8b87fb80c26d kind: Event lastTimestamp: "2026-07-08T07:12:47Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T07:12:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T07:12:47Z" name: model-chainer-raw-hpa-aea5a-66fdcfdbb4-j89r4.18c03f0a9b210040 namespace: kserve-ci-e2e-test resourceVersion: "18100" uid: 8aca014f-947a-4336-a26c-f67fef526de0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-aea5a} kind: Pod name: model-chainer-raw-hpa-aea5a-66fdcfdbb4-j89r4 namespace: kserve-ci-e2e-test resourceVersion: "18085" uid: 398fbe6e-9745-42da-a375-8b87fb80c26d kind: Event lastTimestamp: "2026-07-08T07:12:47Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:343c9e455f840436fd086519b55f7daa1173afbafc1b91e08f3c6de9e52e6594" already present on machine metadata: creationTimestamp: "2026-07-08T07:12:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:12:47Z" name: model-chainer-raw-hpa-aea5a-66fdcfdbb4-j89r4.18c03f0a9c9ea6cf namespace: kserve-ci-e2e-test resourceVersion: "18102" uid: 5ac03af9-cb66-44c1-ae3c-fde2b13f9d9e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-98.ec2.internal source: component: kubelet host: ip-10-0-138-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-aea5a} kind: Pod name: model-chainer-raw-hpa-aea5a-66fdcfdbb4-j89r4 namespace: kserve-ci-e2e-test resourceVersion: "18085" uid: 398fbe6e-9745-42da-a375-8b87fb80c26d kind: Event lastTimestamp: "2026-07-08T07:12:47Z" message: 'Created container: model-chainer-raw-hpa-aea5a' metadata: creationTimestamp: "2026-07-08T07:12:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:12:47Z" name: model-chainer-raw-hpa-aea5a-66fdcfdbb4-j89r4.18c03f0a9f817424 namespace: kserve-ci-e2e-test resourceVersion: "18103" uid: 3fd6ef7f-d8ba-4dc8-b246-65ad46fc987a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-98.ec2.internal source: component: kubelet host: ip-10-0-138-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-aea5a} kind: Pod name: model-chainer-raw-hpa-aea5a-66fdcfdbb4-j89r4 namespace: kserve-ci-e2e-test resourceVersion: "18085" uid: 398fbe6e-9745-42da-a375-8b87fb80c26d kind: Event lastTimestamp: "2026-07-08T07:12:47Z" message: Started container model-chainer-raw-hpa-aea5a metadata: creationTimestamp: "2026-07-08T07:12:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:12:47Z" name: model-chainer-raw-hpa-aea5a-66fdcfdbb4-j89r4.18c03f0a9ff0258f namespace: kserve-ci-e2e-test resourceVersion: "18104" uid: 7df4323d-bb0b-48ff-a237-c61d38779ed5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-98.ec2.internal source: component: kubelet host: ip-10-0-138-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-aea5a} kind: Pod name: model-chainer-raw-hpa-aea5a-66fdcfdbb4-j89r4 namespace: kserve-ci-e2e-test resourceVersion: "18085" uid: 398fbe6e-9745-42da-a375-8b87fb80c26d kind: Event lastTimestamp: "2026-07-08T07:12:56Z" message: Stopping container model-chainer-raw-hpa-aea5a metadata: creationTimestamp: "2026-07-08T07:12:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:12:56Z" name: model-chainer-raw-hpa-aea5a-66fdcfdbb4-j89r4.18c03f0cd8e4facb namespace: kserve-ci-e2e-test resourceVersion: "18191" uid: a3e5b2a4-76e5-49ff-8791-8413438c93d5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-98.ec2.internal source: component: kubelet host: ip-10-0-138-98.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T07:12:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-aea5a} kind: Pod name: model-chainer-raw-hpa-aea5a-66fdcfdbb4-j89r4 namespace: kserve-ci-e2e-test resourceVersion: "18085" uid: 398fbe6e-9745-42da-a375-8b87fb80c26d kind: Event lastTimestamp: "2026-07-08T07:13:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T07:12:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:13:23Z" name: model-chainer-raw-hpa-aea5a-66fdcfdbb4-j89r4.18c03f0d3709f3cd namespace: kserve-ci-e2e-test resourceVersion: "18563" uid: d9b38c14-8d66-4841-81ae-dfc4e4476a9e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-98.ec2.internal source: component: kubelet host: ip-10-0-138-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-aea5a-66fdcfdbb4 namespace: kserve-ci-e2e-test resourceVersion: "18079" uid: 1283e5af-c709-4d4c-9754-0c8677040a24 kind: Event lastTimestamp: "2026-07-08T07:12:46Z" message: 'Created pod: model-chainer-raw-hpa-aea5a-66fdcfdbb4-j89r4' metadata: creationTimestamp: "2026-07-08T07:12:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:12:46Z" name: model-chainer-raw-hpa-aea5a-66fdcfdbb4.18c03f0a80fcbb44 namespace: kserve-ci-e2e-test resourceVersion: "18084" uid: 6111b040-98be-44e0-b446-e60aced51fe6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-aea5a namespace: kserve-ci-e2e-test resourceVersion: "18077" uid: 04fa298f-a103-4bff-ad88-9d4c2737f037 kind: Event lastTimestamp: "2026-07-08T07:12:46Z" message: Scaled up replica set model-chainer-raw-hpa-aea5a-66fdcfdbb4 from 0 to 1 metadata: creationTimestamp: "2026-07-08T07:12:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:12:46Z" name: model-chainer-raw-hpa-aea5a.18c03f0a7ffff2c6 namespace: kserve-ci-e2e-test resourceVersion: "18080" uid: 3d404ee5-a2e4-4aec-9a8a-60a0b27a8b02 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:12:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-aea5a namespace: kserve-ci-e2e-test resourceVersion: "18093" uid: 00db36b3-13c0-426c-b15d-77fca5ad9870 kind: Event lastTimestamp: "2026-07-08T07:12:46Z" message: InferenceGraph [model-chainer-raw-hpa-aea5a] is Ready metadata: creationTimestamp: "2026-07-08T07:12:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:12:46Z" name: model-chainer-raw-hpa-aea5a.18c03f0a851b88b6 namespace: kserve-ci-e2e-test resourceVersion: "18095" uid: e9e5958e-549e-4bbb-a2c7-2ce86f43c2ec reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T07:19:34.305997Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-78661-predictor-bf8977fc5-htw5c namespace: kserve-ci-e2e-test resourceVersion: "21618" uid: 9a1e506b-3a92-49c2-9d3b-a8c16638207e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-78661-predictor-bf8977fc5-htw5c to ip-10-0-135-211.ec2.internal metadata: creationTimestamp: "2026-07-08T07:19:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T07:19:34Z" name: raw-sklearn-78661-predictor-bf8977fc5-htw5c.18c03f6963220c2e namespace: kserve-ci-e2e-test resourceVersion: "21624" uid: 860e6cee-b1b1-45cf-b78d-2ed50d78a8b5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fb64cb696-6mkqx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:34Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-78661-predictor-bf8977fc5-htw5c namespace: kserve-ci-e2e-test resourceVersion: "21628" uid: 9a1e506b-3a92-49c2-9d3b-a8c16638207e kind: Event lastTimestamp: "2026-07-08T07:19:34Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T07:19:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T07:19:34Z" name: raw-sklearn-78661-predictor-bf8977fc5-htw5c.18c03f697cd3f5c1 namespace: kserve-ci-e2e-test resourceVersion: "21636" uid: c48dc19d-421a-492b-a77e-d6fcb7900b65 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-78661-predictor-bf8977fc5-htw5c namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: 9a1e506b-3a92-49c2-9d3b-a8c16638207e kind: Event lastTimestamp: "2026-07-08T07:19:34Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773" already present on machine metadata: creationTimestamp: "2026-07-08T07:19:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:19:34Z" name: raw-sklearn-78661-predictor-bf8977fc5-htw5c.18c03f697e66d800 namespace: kserve-ci-e2e-test resourceVersion: "21638" uid: 99ab29e7-6310-4d2b-aac4-ef3bf22ff039 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-78661-predictor-bf8977fc5-htw5c namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: 9a1e506b-3a92-49c2-9d3b-a8c16638207e kind: Event lastTimestamp: "2026-07-08T07:19:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T07:19:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:19:34Z" name: raw-sklearn-78661-predictor-bf8977fc5-htw5c.18c03f69819a4a69 namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: 6fb5e16b-7b14-461b-9d09-b55285c28fdf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-78661-predictor-bf8977fc5-htw5c namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: 9a1e506b-3a92-49c2-9d3b-a8c16638207e kind: Event lastTimestamp: "2026-07-08T07:19:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T07:19:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:19:34Z" name: raw-sklearn-78661-predictor-bf8977fc5-htw5c.18c03f69820b12c8 namespace: kserve-ci-e2e-test resourceVersion: "21640" uid: 6ee627b6-c0c4-4d56-bf12-3286441b8a56 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-78661-predictor-bf8977fc5-htw5c namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: 9a1e506b-3a92-49c2-9d3b-a8c16638207e kind: Event lastTimestamp: "2026-07-08T07:19:38Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-08T07:19:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:19:38Z" name: raw-sklearn-78661-predictor-bf8977fc5-htw5c.18c03f6a69d5ee61 namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: ed0bad72-2a5d-4322-aee0-1ed914bf463d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-78661-predictor-bf8977fc5-htw5c namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: 9a1e506b-3a92-49c2-9d3b-a8c16638207e kind: Event lastTimestamp: "2026-07-08T07:19:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T07:19:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:19:38Z" name: raw-sklearn-78661-predictor-bf8977fc5-htw5c.18c03f6a6cad12d9 namespace: kserve-ci-e2e-test resourceVersion: "21675" uid: ec720092-5c1e-41f9-80b1-9962d3d38ac9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-78661-predictor-bf8977fc5-htw5c namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: 9a1e506b-3a92-49c2-9d3b-a8c16638207e kind: Event lastTimestamp: "2026-07-08T07:19:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T07:19:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:19:38Z" name: raw-sklearn-78661-predictor-bf8977fc5-htw5c.18c03f6a6d1f38e5 namespace: kserve-ci-e2e-test resourceVersion: "21676" uid: f67b22b2-2edc-49ce-a86d-44240aca3435 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-78661-predictor-bf8977fc5-htw5c namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: 9a1e506b-3a92-49c2-9d3b-a8c16638207e kind: Event lastTimestamp: "2026-07-08T07:19:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T07:19:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:19:38Z" name: raw-sklearn-78661-predictor-bf8977fc5-htw5c.18c03f6a6d28786c namespace: kserve-ci-e2e-test resourceVersion: "21677" uid: 9bd99906-7c0f-46f9-abed-39306c1bfed7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-78661-predictor-bf8977fc5-htw5c namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: 9a1e506b-3a92-49c2-9d3b-a8c16638207e kind: Event lastTimestamp: "2026-07-08T07:19:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T07:19:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:19:38Z" name: raw-sklearn-78661-predictor-bf8977fc5-htw5c.18c03f6a70407121 namespace: kserve-ci-e2e-test resourceVersion: "21681" uid: 4dde5b3d-e70c-466a-adea-b611e99dac11 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-78661-predictor-bf8977fc5-htw5c namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: 9a1e506b-3a92-49c2-9d3b-a8c16638207e kind: Event lastTimestamp: "2026-07-08T07:19:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:19:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:19:38Z" name: raw-sklearn-78661-predictor-bf8977fc5-htw5c.18c03f6a70b05507 namespace: kserve-ci-e2e-test resourceVersion: "21683" uid: 5358cf18-f0b1-49fa-bd94-4f246fbb6434 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T07:19:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-78661-predictor-bf8977fc5-htw5c namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: 9a1e506b-3a92-49c2-9d3b-a8c16638207e kind: Event lastTimestamp: "2026-07-08T07:21:06Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T07:19:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:21:06Z" name: raw-sklearn-78661-predictor-bf8977fc5-htw5c.18c03f6ae1b9b447 namespace: kserve-ci-e2e-test resourceVersion: "22326" uid: 21025064-9b13-4e6c-b9b4-d50120a5c800 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:21:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-78661-predictor-bf8977fc5-htw5c namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: 9a1e506b-3a92-49c2-9d3b-a8c16638207e kind: Event lastTimestamp: "2026-07-08T07:21:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T07:21:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:21:04Z" name: raw-sklearn-78661-predictor-bf8977fc5-htw5c.18c03f7e6071138e namespace: kserve-ci-e2e-test resourceVersion: "22310" uid: 2d71e3d5-3ecf-4111-ae99-06c0cfc1f770 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:21:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-78661-predictor-bf8977fc5-htw5c namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: 9a1e506b-3a92-49c2-9d3b-a8c16638207e kind: Event lastTimestamp: "2026-07-08T07:21:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:21:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:21:04Z" name: raw-sklearn-78661-predictor-bf8977fc5-htw5c.18c03f7e6072a4bb namespace: kserve-ci-e2e-test resourceVersion: "22311" uid: 744b0624-3c67-411e-9e07-ff127de1ac31 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:21:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-78661-predictor-bf8977fc5-htw5c namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: 9a1e506b-3a92-49c2-9d3b-a8c16638207e kind: Event lastTimestamp: "2026-07-08T07:21:06Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T07:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:21:06Z" name: raw-sklearn-78661-predictor-bf8977fc5-htw5c.18c03f7ee7f01327 namespace: kserve-ci-e2e-test resourceVersion: "22325" uid: ab423b70-3cba-4c2e-af28-405f64278e5d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-78661-predictor-bf8977fc5 namespace: kserve-ci-e2e-test resourceVersion: "21609" uid: 4238f4be-d83e-414c-a26b-702ed90fa8a0 kind: Event lastTimestamp: "2026-07-08T07:19:34Z" message: 'Created pod: raw-sklearn-78661-predictor-bf8977fc5-htw5c' metadata: creationTimestamp: "2026-07-08T07:19:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:19:34Z" name: raw-sklearn-78661-predictor-bf8977fc5.18c03f6962956f9b namespace: kserve-ci-e2e-test resourceVersion: "21619" uid: a6de4ac0-6937-4cca-83af-4ca077187e1e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-78661-predictor namespace: kserve-ci-e2e-test resourceVersion: "21607" uid: aaa687ec-e345-4a33-b6d1-c563df80038f kind: Event lastTimestamp: "2026-07-08T07:19:34Z" message: Scaled up replica set raw-sklearn-78661-predictor-bf8977fc5 from 0 to 1 metadata: creationTimestamp: "2026-07-08T07:19:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:19:34Z" name: raw-sklearn-78661-predictor.18c03f695fb90dfb namespace: kserve-ci-e2e-test resourceVersion: "21611" uid: bac481b3-5c9b-43d5-8064-2bc88e4ef5f9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T07:19:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-78661-predictor namespace: kserve-ci-e2e-test resourceVersion: "21608" uid: 09ebaa56-46c6-4a52-95eb-d8f942efd6e0 kind: Event lastTimestamp: "2026-07-08T07:20:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:19:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:20:19Z" name: raw-sklearn-78661-predictor.18c03f6cde8bc0f7 namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: bb209d90-6658-4b6b-91a7-e58d4d9593de reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T07:19:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-78661-predictor namespace: kserve-ci-e2e-test resourceVersion: "21608" uid: 09ebaa56-46c6-4a52-95eb-d8f942efd6e0 kind: Event lastTimestamp: "2026-07-08T07:20:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:19:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:20:19Z" name: raw-sklearn-78661-predictor.18c03f6cdeefb7c2 namespace: kserve-ci-e2e-test resourceVersion: "21968" uid: 48a768b4-037e-43d3-8df9-361049565387 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T07:20:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-78661-predictor namespace: kserve-ci-e2e-test resourceVersion: "21760" uid: 09ebaa56-46c6-4a52-95eb-d8f942efd6e0 kind: Event lastTimestamp: "2026-07-08T07:21:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T07:20:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:21:04Z" name: raw-sklearn-78661-predictor.18c03f775b7bc0db namespace: kserve-ci-e2e-test resourceVersion: "22292" uid: 40247c22-4944-4a64-930a-d492565e43de reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T07:20:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-78661-predictor namespace: kserve-ci-e2e-test resourceVersion: "21760" uid: 09ebaa56-46c6-4a52-95eb-d8f942efd6e0 kind: Event lastTimestamp: "2026-07-08T07:21:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T07:20:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:21:04Z" name: raw-sklearn-78661-predictor.18c03f775bdfb6d4 namespace: kserve-ci-e2e-test resourceVersion: "22294" uid: 5b1466b1-5c3d-4e9c-834c-ac2486d39018 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T07:19:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-78661 namespace: kserve-ci-e2e-test resourceVersion: "21599" uid: 42789a63-73da-466b-9dcb-ac347a40cdad kind: Event lastTimestamp: "2026-07-08T07:20:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T07:19:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:20:56Z" name: raw-sklearn-78661.18c03f695fc7ad37 namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: a1aaf343-10af-487b-ab5d-46aff4d2e282 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:19:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-78661 namespace: kserve-ci-e2e-test resourceVersion: "21599" uid: 42789a63-73da-466b-9dcb-ac347a40cdad kind: Event lastTimestamp: "2026-07-08T07:19:34Z" message: 'Failed to update status for InferenceService "raw-sklearn-78661": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-78661": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T07:19:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:19:34Z" name: raw-sklearn-78661.18c03f696051167b namespace: kserve-ci-e2e-test resourceVersion: "21615" uid: d10d3a31-311d-4e62-9104-43264fa1812f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:20:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-78661 namespace: kserve-ci-e2e-test resourceVersion: "22235" uid: 42789a63-73da-466b-9dcb-ac347a40cdad kind: Event lastTimestamp: "2026-07-08T07:20:56Z" message: InferenceService [raw-sklearn-78661] is Ready metadata: creationTimestamp: "2026-07-08T07:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:20:56Z" name: raw-sklearn-78661.18c03f7c99741003 namespace: kserve-ci-e2e-test resourceVersion: "22236" uid: 74dd6462-2b56-44f0-9a31-1b96b9383dac reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T07:21:09.607444Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b namespace: kserve-ci-e2e-test resourceVersion: "22372" uid: e2ac5022-59fc-4636-9d09-dc8e7ec76662 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b to ip-10-0-135-211.ec2.internal metadata: creationTimestamp: "2026-07-08T07:21:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T07:21:09Z" name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b.18c03f7f938aaebb namespace: kserve-ci-e2e-test resourceVersion: "22376" uid: 220d0554-e81f-485a-8daf-07085406a658 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fb64cb696-6mkqx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:21:09Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b namespace: kserve-ci-e2e-test resourceVersion: "22374" uid: e2ac5022-59fc-4636-9d09-dc8e7ec76662 kind: Event lastTimestamp: "2026-07-08T07:21:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-runtime-24704-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T07:21:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:21:09Z" name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b.18c03f7f9b9d9997 namespace: kserve-ci-e2e-test resourceVersion: "22383" uid: 6c1e7e7d-4bab-4c50-8a16-c725cae3c5db reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:21:10Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b namespace: kserve-ci-e2e-test resourceVersion: "22380" uid: e2ac5022-59fc-4636-9d09-dc8e7ec76662 kind: Event lastTimestamp: "2026-07-08T07:21:10Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T07:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T07:21:10Z" name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b.18c03f7fd0c4ba0f namespace: kserve-ci-e2e-test resourceVersion: "22395" uid: 328bdf98-b062-4559-a8f4-f6e5edecf0e1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:21:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b namespace: kserve-ci-e2e-test resourceVersion: "22374" uid: e2ac5022-59fc-4636-9d09-dc8e7ec76662 kind: Event lastTimestamp: "2026-07-08T07:21:10Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773" already present on machine metadata: creationTimestamp: "2026-07-08T07:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:21:10Z" name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b.18c03f7fd231ff2e namespace: kserve-ci-e2e-test resourceVersion: "22397" uid: 085a9d16-c8d0-4370-af59-d8a9f627e189 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:21:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b namespace: kserve-ci-e2e-test resourceVersion: "22374" uid: e2ac5022-59fc-4636-9d09-dc8e7ec76662 kind: Event lastTimestamp: "2026-07-08T07:21:10Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T07:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:21:10Z" name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b.18c03f7fd52698e0 namespace: kserve-ci-e2e-test resourceVersion: "22399" uid: 11fc2b1e-df3c-444e-adbd-731e0bf17215 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:21:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b namespace: kserve-ci-e2e-test resourceVersion: "22374" uid: e2ac5022-59fc-4636-9d09-dc8e7ec76662 kind: Event lastTimestamp: "2026-07-08T07:21:10Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T07:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:21:10Z" name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b.18c03f7fd5b75f32 namespace: kserve-ci-e2e-test resourceVersion: "22401" uid: c80bdb93-c6d3-4709-b1dd-7e2d1970ea3a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:21:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b namespace: kserve-ci-e2e-test resourceVersion: "22374" uid: e2ac5022-59fc-4636-9d09-dc8e7ec76662 kind: Event lastTimestamp: "2026-07-08T07:21:15Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-08T07:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:21:15Z" name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b.18c03f80db1a6551 namespace: kserve-ci-e2e-test resourceVersion: "22441" uid: 61628d7f-8dbd-41f9-9c1f-73b16a24e699 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:21:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b namespace: kserve-ci-e2e-test resourceVersion: "22374" uid: e2ac5022-59fc-4636-9d09-dc8e7ec76662 kind: Event lastTimestamp: "2026-07-08T07:21:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T07:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:21:15Z" name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b.18c03f80de1989cb namespace: kserve-ci-e2e-test resourceVersion: "22443" uid: d72d505a-b563-4544-bbd2-38763195c867 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:21:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b namespace: kserve-ci-e2e-test resourceVersion: "22374" uid: e2ac5022-59fc-4636-9d09-dc8e7ec76662 kind: Event lastTimestamp: "2026-07-08T07:21:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T07:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:21:15Z" name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b.18c03f80de8d1b70 namespace: kserve-ci-e2e-test resourceVersion: "22446" uid: a9b01d8e-11fe-4839-8a7f-6ed555378cb5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:21:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b namespace: kserve-ci-e2e-test resourceVersion: "22374" uid: e2ac5022-59fc-4636-9d09-dc8e7ec76662 kind: Event lastTimestamp: "2026-07-08T07:21:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T07:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:21:15Z" name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b.18c03f80de9561ca namespace: kserve-ci-e2e-test resourceVersion: "22447" uid: db3ce2da-2f06-41b9-a9d2-84a1e271e328 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:21:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b namespace: kserve-ci-e2e-test resourceVersion: "22374" uid: e2ac5022-59fc-4636-9d09-dc8e7ec76662 kind: Event lastTimestamp: "2026-07-08T07:21:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T07:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:21:15Z" name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b.18c03f80e186d52d namespace: kserve-ci-e2e-test resourceVersion: "22449" uid: 4ad14618-dde8-4e61-abfb-a65424592e44 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:21:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b namespace: kserve-ci-e2e-test resourceVersion: "22374" uid: e2ac5022-59fc-4636-9d09-dc8e7ec76662 kind: Event lastTimestamp: "2026-07-08T07:21:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:21:15Z" name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b.18c03f80e1fa7450 namespace: kserve-ci-e2e-test resourceVersion: "22450" uid: 209c34a2-c975-46f1-8f6d-fce272c2c997 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T07:21:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b namespace: kserve-ci-e2e-test resourceVersion: "22374" uid: e2ac5022-59fc-4636-9d09-dc8e7ec76662 kind: Event lastTimestamp: "2026-07-08T07:22:43Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T07:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:22:43Z" name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b.18c03f8153050879 namespace: kserve-ci-e2e-test resourceVersion: "23099" uid: f7592a61-367d-4076-9b98-0ff1c51b8946 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:22:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b namespace: kserve-ci-e2e-test resourceVersion: "22374" uid: e2ac5022-59fc-4636-9d09-dc8e7ec76662 kind: Event lastTimestamp: "2026-07-08T07:22:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T07:22:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:22:39Z" name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b.18c03f948e0f2ac3 namespace: kserve-ci-e2e-test resourceVersion: "23066" uid: 9623180a-9ffa-4da1-91b6-ce39595f4d10 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:22:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b namespace: kserve-ci-e2e-test resourceVersion: "22374" uid: e2ac5022-59fc-4636-9d09-dc8e7ec76662 kind: Event lastTimestamp: "2026-07-08T07:22:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T07:22:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:22:39Z" name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b.18c03f948e10a1a6 namespace: kserve-ci-e2e-test resourceVersion: "23067" uid: 819594bf-8dbf-40d9-b460-c55b0e68e0e7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:22:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b namespace: kserve-ci-e2e-test resourceVersion: "22374" uid: e2ac5022-59fc-4636-9d09-dc8e7ec76662 kind: Event lastTimestamp: "2026-07-08T07:22:43Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T07:22:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T07:22:43Z" name: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b.18c03f9559390d54 namespace: kserve-ci-e2e-test resourceVersion: "23098" uid: 2e80adee-158e-47ef-8299-d64c24a6e120 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-211.ec2.internal source: component: kubelet host: ip-10-0-135-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:21:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-24704-predictor-656b49b9c5 namespace: kserve-ci-e2e-test resourceVersion: "22360" uid: 971849f5-cb19-4f1e-bfea-8cae2258129c kind: Event lastTimestamp: "2026-07-08T07:21:09Z" message: 'Created pod: raw-sklearn-runtime-24704-predictor-656b49b9c5-xf97b' metadata: creationTimestamp: "2026-07-08T07:21:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:21:09Z" name: raw-sklearn-runtime-24704-predictor-656b49b9c5.18c03f7f9320754f namespace: kserve-ci-e2e-test resourceVersion: "22373" uid: 67676837-210d-4c20-b9b1-3e2d41b74c39 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:21:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-24704-predictor namespace: kserve-ci-e2e-test resourceVersion: "22358" uid: 82ce95d6-de62-4d7a-881a-5ef98274dafb kind: Event lastTimestamp: "2026-07-08T07:21:09Z" message: Scaled up replica set raw-sklearn-runtime-24704-predictor-656b49b9c5 from 0 to 1 metadata: creationTimestamp: "2026-07-08T07:21:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:21:09Z" name: raw-sklearn-runtime-24704-predictor.18c03f7f8ea3290c namespace: kserve-ci-e2e-test resourceVersion: "22361" uid: ed6cd0de-178b-4923-9e1d-e1493545d9b0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:21:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-24704-predictor namespace: kserve-ci-e2e-test resourceVersion: "22359" uid: b688c807-989f-4078-ba33-26268872015a kind: Event lastTimestamp: "2026-07-08T07:21:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:21:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:21:39Z" name: raw-sklearn-runtime-24704-predictor.18c03f830d789d62 namespace: kserve-ci-e2e-test resourceVersion: "22624" uid: dbac0218-ca50-4142-a1d0-41442574c90b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T07:21:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-24704-predictor namespace: kserve-ci-e2e-test resourceVersion: "22359" uid: b688c807-989f-4078-ba33-26268872015a kind: Event lastTimestamp: "2026-07-08T07:21:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T07:21:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:21:39Z" name: raw-sklearn-runtime-24704-predictor.18c03f830dd732ea namespace: kserve-ci-e2e-test resourceVersion: "22625" uid: 7d65177f-efd1-43ad-b932-f8ba881c0ec5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T07:21:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-24704-predictor namespace: kserve-ci-e2e-test resourceVersion: "22522" uid: b688c807-989f-4078-ba33-26268872015a kind: Event lastTimestamp: "2026-07-08T07:22:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T07:21:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:22:39Z" name: raw-sklearn-runtime-24704-predictor.18c03f8a0b997cd6 namespace: kserve-ci-e2e-test resourceVersion: "23050" uid: ae6edc26-3069-44a1-8165-624049b45e04 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T07:21:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-24704-predictor namespace: kserve-ci-e2e-test resourceVersion: "22522" uid: b688c807-989f-4078-ba33-26268872015a kind: Event lastTimestamp: "2026-07-08T07:22:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T07:21:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T07:22:39Z" name: raw-sklearn-runtime-24704-predictor.18c03f8a0c00ab0d namespace: kserve-ci-e2e-test resourceVersion: "23051" uid: 9a476577-fa94-45c8-a730-96428f37739d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-08T07:21:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-24704 namespace: kserve-ci-e2e-test resourceVersion: "22351" uid: c9944feb-d4ef-41aa-b451-274df5cc8395 kind: Event lastTimestamp: "2026-07-08T07:22:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T07:21:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:22:33Z" name: raw-sklearn-runtime-24704.18c03f7f8edd1668 namespace: kserve-ci-e2e-test resourceVersion: "23008" uid: be296d2b-e118-40d4-a411-33e6972ae69b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:21:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-24704 namespace: kserve-ci-e2e-test resourceVersion: "22351" uid: c9944feb-d4ef-41aa-b451-274df5cc8395 kind: Event lastTimestamp: "2026-07-08T07:21:09Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-24704": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-24704": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T07:21:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:21:09Z" name: raw-sklearn-runtime-24704.18c03f7f8f395668 namespace: kserve-ci-e2e-test resourceVersion: "22364" uid: df857af2-45a1-42d4-84c2-993bd0e61254 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T07:22:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-24704 namespace: kserve-ci-e2e-test resourceVersion: "23006" uid: c9944feb-d4ef-41aa-b451-274df5cc8395 kind: Event lastTimestamp: "2026-07-08T07:22:33Z" message: InferenceService [raw-sklearn-runtime-24704] is Ready metadata: creationTimestamp: "2026-07-08T07:22:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T07:22:33Z" name: raw-sklearn-runtime-24704.18c03f930ac815ef namespace: kserve-ci-e2e-test resourceVersion: "23007" uid: 8a72e986-b5ca-4aff-ab34-7b70369903f2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "23316"