--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:11:47.619912Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-4ed3d4-predictor-7b4f64db57-tt5xh namespace: kserve-ci-e2e-test resourceVersion: "21234" uid: 11a0ecfe-5f06-4878-a71f-8ac19e112e80 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-4ed3d4-predictor-7b4f64db57-tt5xh to ip-10-0-135-167.ec2.internal metadata: creationTimestamp: "2026-06-15T06:11:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:11:47Z" name: isvc-init-fail-4ed3d4-predictor-7b4f64db57-tt5xh.18b92c5d7ba7d870 namespace: kserve-ci-e2e-test resourceVersion: "21240" uid: e0a6acd9-6550-45c0-919c-d28f657ca5cb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65bc5545-rprtl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:48Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-4ed3d4-predictor-7b4f64db57-tt5xh namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 11a0ecfe-5f06-4878-a71f-8ac19e112e80 kind: Event lastTimestamp: "2026-06-15T06:11:48Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:11:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:11:48Z" name: isvc-init-fail-4ed3d4-predictor-7b4f64db57-tt5xh.18b92c5d965785a1 namespace: kserve-ci-e2e-test resourceVersion: "21257" uid: 7916e61a-ff5f-49f1-8c98-4e109786f71e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:11:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-4ed3d4-predictor-7b4f64db57-tt5xh namespace: kserve-ci-e2e-test resourceVersion: "21238" uid: 11a0ecfe-5f06-4878-a71f-8ac19e112e80 kind: Event lastTimestamp: "2026-06-15T06:11:51Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ba8edcbfb3f9312d158be16483785d7654e60c7090f262c42214fd2b29effada" already present on machine metadata: creationTimestamp: "2026-06-15T06:11:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:51Z" name: isvc-init-fail-4ed3d4-predictor-7b4f64db57-tt5xh.18b92c5d97bd33e9 namespace: kserve-ci-e2e-test resourceVersion: "21292" uid: 6f770b9d-5141-40ad-b701-055f26f80e93 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:11:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-4ed3d4-predictor-7b4f64db57-tt5xh namespace: kserve-ci-e2e-test resourceVersion: "21238" uid: 11a0ecfe-5f06-4878-a71f-8ac19e112e80 kind: Event lastTimestamp: "2026-06-15T06:11:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-15T06:11:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:51Z" name: isvc-init-fail-4ed3d4-predictor-7b4f64db57-tt5xh.18b92c5d9a502ead namespace: kserve-ci-e2e-test resourceVersion: "21296" uid: e2eb1f28-4ee6-4e4d-9066-f7605687aa83 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:11:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-4ed3d4-predictor-7b4f64db57-tt5xh namespace: kserve-ci-e2e-test resourceVersion: "21238" uid: 11a0ecfe-5f06-4878-a71f-8ac19e112e80 kind: Event lastTimestamp: "2026-06-15T06:11:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-15T06:11:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:51Z" name: isvc-init-fail-4ed3d4-predictor-7b4f64db57-tt5xh.18b92c5d9ac71695 namespace: kserve-ci-e2e-test resourceVersion: "21297" uid: 4c4879b7-81e1-442a-9bdb-224d16b828d2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-4ed3d4-predictor-7b4f64db57-tt5xh namespace: kserve-ci-e2e-test resourceVersion: "21238" uid: 11a0ecfe-5f06-4878-a71f-8ac19e112e80 kind: Event lastTimestamp: "2026-06-15T06:11:53Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-06-15T06:11:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:53Z" name: isvc-init-fail-4ed3d4-predictor-7b4f64db57-tt5xh.18b92c5ed7647b07 namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: b41138de-9be6-4882-bf98-0e2f52e027a9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-4ed3d4-predictor-7b4f64db57 namespace: kserve-ci-e2e-test resourceVersion: "21225" uid: 547ed7e2-8f31-479f-a233-12d68baddadc kind: Event lastTimestamp: "2026-06-15T06:11:47Z" message: 'Created pod: isvc-init-fail-4ed3d4-predictor-7b4f64db57-tt5xh' metadata: creationTimestamp: "2026-06-15T06:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:47Z" name: isvc-init-fail-4ed3d4-predictor-7b4f64db57.18b92c5d7b1796ed namespace: kserve-ci-e2e-test resourceVersion: "21237" uid: 8f2aa86f-d784-4496-b06b-83390170157e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-4ed3d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 900f4b5f-4049-490b-bfd5-7905082956d3 kind: Event lastTimestamp: "2026-06-15T06:11:47Z" message: Scaled up replica set isvc-init-fail-4ed3d4-predictor-7b4f64db57 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:47Z" name: isvc-init-fail-4ed3d4-predictor.18b92c5d77c28dac namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: cd24ae93-a432-46ba-9bd5-8bb51c101dd6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-06-15T06:11:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-4ed3d4 namespace: kserve-ci-e2e-test resourceVersion: "21196" uid: ca356f5c-58f8-47a5-a4bd-4d36bf608b89 kind: Event lastTimestamp: "2026-06-15T06:11:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:51Z" name: isvc-init-fail-4ed3d4.18b92c5d7720b945 namespace: kserve-ci-e2e-test resourceVersion: "21300" uid: 957f64c2-3694-4b1f-9783-aee67b1e5f19 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:11:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-4ed3d4 namespace: kserve-ci-e2e-test resourceVersion: "21196" uid: ca356f5c-58f8-47a5-a4bd-4d36bf608b89 kind: Event lastTimestamp: "2026-06-15T06:11:47Z" message: 'Failed to update status for InferenceService "isvc-init-fail-4ed3d4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-4ed3d4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:47Z" name: isvc-init-fail-4ed3d4.18b92c5d77a744da namespace: kserve-ci-e2e-test resourceVersion: "21248" uid: 471f49db-e9d5-4861-8dda-09d0f4404d90 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:11:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-4ed3d4 namespace: kserve-ci-e2e-test resourceVersion: "21196" uid: ca356f5c-58f8-47a5-a4bd-4d36bf608b89 kind: Event lastTimestamp: "2026-06-15T06:11:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-4ed3d4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:47Z" name: isvc-init-fail-4ed3d4.18b92c5d77a7a7ba namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 07c44206-fc24-4b3f-916a-690432e92fe7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:05:33.642291Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 namespace: kserve-ci-e2e-test resourceVersion: "18201" uid: dd2bb2e7-2f13-4183-8c96-e8157ab031a3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 to ip-10-0-135-167.ec2.internal metadata: creationTimestamp: "2026-06-15T06:05:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:05:33Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2.18b92c0668da2e1a namespace: kserve-ci-e2e-test resourceVersion: "18205" uid: 3209c799-28f8-4b6d-9ac1-f4c5b4cf2135 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65bc5545-rprtl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:34Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 namespace: kserve-ci-e2e-test resourceVersion: "18209" uid: dd2bb2e7-2f13-4183-8c96-e8157ab031a3 kind: Event lastTimestamp: "2026-06-15T06:05:34Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:05:34Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2.18b92c06829efc40 namespace: kserve-ci-e2e-test resourceVersion: "18216" uid: 7bd2569d-05c1-46b4-bc26-f7e48e65e71a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: dd2bb2e7-2f13-4183-8c96-e8157ab031a3 kind: Event lastTimestamp: "2026-06-15T06:05:34Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ba8edcbfb3f9312d158be16483785d7654e60c7090f262c42214fd2b29effada" already present on machine metadata: creationTimestamp: "2026-06-15T06:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:34Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2.18b92c0684976fd7 namespace: kserve-ci-e2e-test resourceVersion: "18218" uid: 107ae3e6-607d-4254-90e2-dbc26d26cc50 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: dd2bb2e7-2f13-4183-8c96-e8157ab031a3 kind: Event lastTimestamp: "2026-06-15T06:05:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-15T06:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:34Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2.18b92c0687bd4e21 namespace: kserve-ci-e2e-test resourceVersion: "18219" uid: 5147317c-8834-481c-841d-0756ace9693a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: dd2bb2e7-2f13-4183-8c96-e8157ab031a3 kind: Event lastTimestamp: "2026-06-15T06:05:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-15T06:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:34Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2.18b92c068836d8a6 namespace: kserve-ci-e2e-test resourceVersion: "18220" uid: 902c2775-fa12-4090-89e9-a46683c8dfd5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: dd2bb2e7-2f13-4183-8c96-e8157ab031a3 kind: Event lastTimestamp: "2026-06-15T06:05:38Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:38Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2.18b92c076e176832 namespace: kserve-ci-e2e-test resourceVersion: "18257" uid: 579f8304-ec24-4ebe-9e16-6b806c0b6190 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: dd2bb2e7-2f13-4183-8c96-e8157ab031a3 kind: Event lastTimestamp: "2026-06-15T06:05:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:38Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2.18b92c077134a1a5 namespace: kserve-ci-e2e-test resourceVersion: "18259" uid: 01c6f4f6-f13a-4442-bc8b-4b653ff1b86f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: dd2bb2e7-2f13-4183-8c96-e8157ab031a3 kind: Event lastTimestamp: "2026-06-15T06:05:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:38Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2.18b92c0771aba4b5 namespace: kserve-ci-e2e-test resourceVersion: "18262" uid: d8eb5af8-1ffb-4710-8e0d-316582bc851e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: dd2bb2e7-2f13-4183-8c96-e8157ab031a3 kind: Event lastTimestamp: "2026-06-15T06:05:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:38Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2.18b92c0771b69e4d namespace: kserve-ci-e2e-test resourceVersion: "18263" uid: 5dd778ea-cc8f-47bc-8dc4-5fc829638af3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: dd2bb2e7-2f13-4183-8c96-e8157ab031a3 kind: Event lastTimestamp: "2026-06-15T06:05:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:38Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2.18b92c077520ca59 namespace: kserve-ci-e2e-test resourceVersion: "18265" uid: cef03169-83e4-4a61-857e-5ba6cf931991 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: dd2bb2e7-2f13-4183-8c96-e8157ab031a3 kind: Event lastTimestamp: "2026-06-15T06:05:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:38Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2.18b92c07759a0052 namespace: kserve-ci-e2e-test resourceVersion: "18266" uid: f3baad3c-0d10-40fd-9231-18da13d135ac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: dd2bb2e7-2f13-4183-8c96-e8157ab031a3 kind: Event lastTimestamp: "2026-06-15T06:05:38Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:41e9c610ce929fc56f4076f97bf42d251460cf8b6a90f49e0a84e5b92946522f" already present on machine metadata: creationTimestamp: "2026-06-15T06:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:38Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2.18b92c0775a49a80 namespace: kserve-ci-e2e-test resourceVersion: "18267" uid: f6b3d0e7-3e7c-46f4-86b3-61e9545e9f98 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: dd2bb2e7-2f13-4183-8c96-e8157ab031a3 kind: Event lastTimestamp: "2026-06-15T06:05:38Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-06-15T06:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:38Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2.18b92c07789820a4 namespace: kserve-ci-e2e-test resourceVersion: "18268" uid: 7752c3c9-a15a-437d-a79d-f1f13c0adfa0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: dd2bb2e7-2f13-4183-8c96-e8157ab031a3 kind: Event lastTimestamp: "2026-06-15T06:05:38Z" message: Started container agent metadata: creationTimestamp: "2026-06-15T06:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:38Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2.18b92c07790d7bb4 namespace: kserve-ci-e2e-test resourceVersion: "18269" uid: f155df8e-3966-47a4-b83d-e3b2b6bc6663 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-06-15T06:05:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: dd2bb2e7-2f13-4183-8c96-e8157ab031a3 kind: Event lastTimestamp: "2026-06-15T06:07:16Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:07:16Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2.18b92c07aa5d8a25 namespace: kserve-ci-e2e-test resourceVersion: "19093" uid: 38a13847-e279-483a-b758-92ee88f103c9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-06-15T06:05:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: dd2bb2e7-2f13-4183-8c96-e8157ab031a3 kind: Event lastTimestamp: "2026-06-15T06:07:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-06-15T06:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:07:16Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2.18b92c07e62e888f namespace: kserve-ci-e2e-test resourceVersion: "19094" uid: 1abaee9b-1306-4927-bf3e-ec40d8db7795 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: dd2bb2e7-2f13-4183-8c96-e8157ab031a3 kind: Event lastTimestamp: "2026-06-15T06:07:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:07:08Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2.18b92c1c94bf4134 namespace: kserve-ci-e2e-test resourceVersion: "18977" uid: 1aabb685-0f9a-4278-a1e6-35df3e93636c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: dd2bb2e7-2f13-4183-8c96-e8157ab031a3 kind: Event lastTimestamp: "2026-06-15T06:07:08Z" message: Stopping container agent metadata: creationTimestamp: "2026-06-15T06:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:07:08Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2.18b92c1c94bf7f41 namespace: kserve-ci-e2e-test resourceVersion: "18978" uid: b3472d0d-ddad-408f-8300-c5679d4a2ee8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: dd2bb2e7-2f13-4183-8c96-e8157ab031a3 kind: Event lastTimestamp: "2026-06-15T06:07:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:07:08Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2.18b92c1c94bf9275 namespace: kserve-ci-e2e-test resourceVersion: "18980" uid: 4a135d8b-da8a-4309-88d8-9bbb73bcb076 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:07:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2 namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: dd2bb2e7-2f13-4183-8c96-e8157ab031a3 kind: Event lastTimestamp: "2026-06-15T06:07:26Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:07:26Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2.18b92c1d166a4ac9 namespace: kserve-ci-e2e-test resourceVersion: "19168" uid: 6f5d2832-56f4-4d5f-98f1-bfe5ca7b47c9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-e564c-predictor-566bdb64cc namespace: kserve-ci-e2e-test resourceVersion: "18190" uid: 48ae2e8d-5a30-4b2a-86f6-f4729826dc4f kind: Event lastTimestamp: "2026-06-15T06:05:33Z" message: 'Created pod: isvc-logger-raw-e564c-predictor-566bdb64cc-mnsp2' metadata: creationTimestamp: "2026-06-15T06:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:05:33Z" name: isvc-logger-raw-e564c-predictor-566bdb64cc.18b92c06684e0f9a namespace: kserve-ci-e2e-test resourceVersion: "18202" uid: 2accc612-1e6a-495c-9113-990f271cbedf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-e564c-predictor namespace: kserve-ci-e2e-test resourceVersion: "18188" uid: 709730e5-70a3-42aa-8fc7-948e4da0b0d0 kind: Event lastTimestamp: "2026-06-15T06:05:33Z" message: Scaled up replica set isvc-logger-raw-e564c-predictor-566bdb64cc from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:05:33Z" name: isvc-logger-raw-e564c-predictor.18b92c06648d565d namespace: kserve-ci-e2e-test resourceVersion: "18192" uid: 150610f3-095e-43a5-a320-d6169771d047 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-e564c-predictor namespace: kserve-ci-e2e-test resourceVersion: "18189" uid: a37275ae-2ed8-40ce-acde-5a4fc48e6029 kind: Event lastTimestamp: "2026-06-15T06:05:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:05:48Z" name: isvc-logger-raw-e564c-predictor.18b92c09e303de3e namespace: kserve-ci-e2e-test resourceVersion: "18356" uid: f9a6097b-fb39-4903-83ed-6646b0e9275f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-e564c-predictor namespace: kserve-ci-e2e-test resourceVersion: "18189" uid: a37275ae-2ed8-40ce-acde-5a4fc48e6029 kind: Event lastTimestamp: "2026-06-15T06:05:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:05:48Z" name: isvc-logger-raw-e564c-predictor.18b92c09e3627b75 namespace: kserve-ci-e2e-test resourceVersion: "18358" uid: bdf96e7a-3889-42ee-a4c1-c71e552956da reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:06:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-e564c-predictor namespace: kserve-ci-e2e-test resourceVersion: "18357" uid: a37275ae-2ed8-40ce-acde-5a4fc48e6029 kind: Event lastTimestamp: "2026-06-15T06:07:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:07:03Z" name: isvc-logger-raw-e564c-predictor.18b92c0d6257531a namespace: kserve-ci-e2e-test resourceVersion: "18898" uid: e3fc8731-7ae5-4222-9ebd-dc774f7a085e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:06:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-e564c-predictor namespace: kserve-ci-e2e-test resourceVersion: "18357" uid: a37275ae-2ed8-40ce-acde-5a4fc48e6029 kind: Event lastTimestamp: "2026-06-15T06:07:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:07:03Z" name: isvc-logger-raw-e564c-predictor.18b92c0d62c0b0b6 namespace: kserve-ci-e2e-test resourceVersion: "18899" uid: b7197b3d-a304-449b-ade5-4600d42dbfb2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-06-15T06:05:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-e564c namespace: kserve-ci-e2e-test resourceVersion: "18179" uid: 6ab15b35-fa43-46be-be58-67a39cd0dd20 kind: Event lastTimestamp: "2026-06-15T06:06:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:06:56Z" name: isvc-logger-raw-e564c.18b92c06646bc502 namespace: kserve-ci-e2e-test resourceVersion: "18852" uid: 0d92609f-5bb7-47f6-9358-c90dfec41eb9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-e564c namespace: kserve-ci-e2e-test resourceVersion: "18179" uid: 6ab15b35-fa43-46be-be58-67a39cd0dd20 kind: Event lastTimestamp: "2026-06-15T06:05:33Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-e564c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-e564c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:05:33Z" name: isvc-logger-raw-e564c.18b92c0664eae5d5 namespace: kserve-ci-e2e-test resourceVersion: "18195" uid: 774029f7-03ba-4b7e-9c4e-75fe414219a5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:06:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-e564c namespace: kserve-ci-e2e-test resourceVersion: "18850" uid: 6ab15b35-fa43-46be-be58-67a39cd0dd20 kind: Event lastTimestamp: "2026-06-15T06:06:56Z" message: InferenceService [isvc-logger-raw-e564c] is Ready metadata: creationTimestamp: "2026-06-15T06:06:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:06:56Z" name: isvc-logger-raw-e564c.18b92c199e831bb9 namespace: kserve-ci-e2e-test resourceVersion: "18851" uid: c984dd81-3685-4ed8-bc9c-d3bea789bdc7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:09:59.204262Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc namespace: kserve-ci-e2e-test resourceVersion: "20291" uid: 98f8fee9-da67-4069-a0e4-738cb7dc7a9c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc to ip-10-0-135-167.ec2.internal metadata: creationTimestamp: "2026-06-15T06:09:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:09:59Z" name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc.18b92c443d944bf0 namespace: kserve-ci-e2e-test resourceVersion: "20296" uid: 9636fda5-2576-4917-a912-af9a5a372f9b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65bc5545-rprtl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:09:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 98f8fee9-da67-4069-a0e4-738cb7dc7a9c kind: Event lastTimestamp: "2026-06-15T06:09:59Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-primary-f19eac-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:09:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:09:59Z" name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc.18b92c4446ba54dd namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: 9a1649b6-481b-42be-9e81-9ee2cf7df513 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:10:00Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc namespace: kserve-ci-e2e-test resourceVersion: "20300" uid: 98f8fee9-da67-4069-a0e4-738cb7dc7a9c kind: Event lastTimestamp: "2026-06-15T06:10:00Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:10:00Z" name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc.18b92c447b0cf21f namespace: kserve-ci-e2e-test resourceVersion: "20316" uid: 3caa9aae-edb5-4aa2-8b79-f4fbe4cffde6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:10:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 98f8fee9-da67-4069-a0e4-738cb7dc7a9c kind: Event lastTimestamp: "2026-06-15T06:10:00Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ba8edcbfb3f9312d158be16483785d7654e60c7090f262c42214fd2b29effada" already present on machine metadata: creationTimestamp: "2026-06-15T06:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:10:00Z" name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc.18b92c447c5bf6c5 namespace: kserve-ci-e2e-test resourceVersion: "20318" uid: f120b2d7-3f53-4e34-9263-dca954c43a6f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:10:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 98f8fee9-da67-4069-a0e4-738cb7dc7a9c kind: Event lastTimestamp: "2026-06-15T06:10:00Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-15T06:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:10:00Z" name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc.18b92c447fea2407 namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: 86e7e441-1b75-4483-90fc-3bb41e8b28ea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:10:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 98f8fee9-da67-4069-a0e4-738cb7dc7a9c kind: Event lastTimestamp: "2026-06-15T06:10:00Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-15T06:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:10:00Z" name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc.18b92c44806515f5 namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: 71e167ff-c5b6-4f68-b909-bf6ac7dfe19a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:10:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 98f8fee9-da67-4069-a0e4-738cb7dc7a9c kind: Event lastTimestamp: "2026-06-15T06:10:05Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:10:05Z" name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc.18b92c4598f56aa7 namespace: kserve-ci-e2e-test resourceVersion: "20357" uid: 1717c154-3f26-48bb-a1e3-f0ada6fe3504 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:10:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 98f8fee9-da67-4069-a0e4-738cb7dc7a9c kind: Event lastTimestamp: "2026-06-15T06:10:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:10:05Z" name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc.18b92c459c994141 namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: 3f1add7e-a4c8-4506-a572-ca3b37f3b7a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:10:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 98f8fee9-da67-4069-a0e4-738cb7dc7a9c kind: Event lastTimestamp: "2026-06-15T06:10:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:10:05Z" name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc.18b92c459d188e37 namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 461a51a7-ea56-426b-a44c-ef1f63dfea6a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:10:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 98f8fee9-da67-4069-a0e4-738cb7dc7a9c kind: Event lastTimestamp: "2026-06-15T06:10:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:10:05Z" name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc.18b92c459d2457ed namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: 3dbfe5a8-2cda-4563-8f62-1e6806b1b33d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:10:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 98f8fee9-da67-4069-a0e4-738cb7dc7a9c kind: Event lastTimestamp: "2026-06-15T06:10:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:10:05Z" name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc.18b92c45a0611064 namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: 2be6f755-e9cb-454b-9953-561070241b80 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:10:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 98f8fee9-da67-4069-a0e4-738cb7dc7a9c kind: Event lastTimestamp: "2026-06-15T06:10:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:10:05Z" name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc.18b92c45a0ed8d2f namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: 0a15a305-0ab7-4965-9661-0edebc392dbe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-06-15T06:10:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 98f8fee9-da67-4069-a0e4-738cb7dc7a9c kind: Event lastTimestamp: "2026-06-15T06:11:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:13Z" name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc.18b92c4610cd12fe namespace: kserve-ci-e2e-test resourceVersion: "20856" uid: 8949ca28-e8e9-4551-a980-58ab5cb454bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 98f8fee9-da67-4069-a0e4-738cb7dc7a9c kind: Event lastTimestamp: "2026-06-15T06:11:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:47Z" name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc.18b92c5d75c41ea7 namespace: kserve-ci-e2e-test resourceVersion: "21218" uid: 1efb182d-2f1b-4bae-a730-72b3d3541505 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 98f8fee9-da67-4069-a0e4-738cb7dc7a9c kind: Event lastTimestamp: "2026-06-15T06:11:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:47Z" name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc.18b92c5d75c6a92f namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: d50af6d2-dccc-4ed8-b3a2-b9cc8ab714e3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 98f8fee9-da67-4069-a0e4-738cb7dc7a9c kind: Event lastTimestamp: "2026-06-15T06:11:48Z" message: 'Readiness probe failed: Get "https://10.132.0.27:8643/healthz": dial tcp 10.132.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:11:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:48Z" name: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc.18b92c5d9515053b namespace: kserve-ci-e2e-test resourceVersion: "21256" uid: d1d41cf7-96a6-44ad-90dd-fd9c4a597c36 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:09:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-f19eac-predictor-6ff95b59f6 namespace: kserve-ci-e2e-test resourceVersion: "20280" uid: c63c91da-bcc5-477d-aae4-983ebae39870 kind: Event lastTimestamp: "2026-06-15T06:09:59Z" message: 'Created pod: isvc-primary-f19eac-predictor-6ff95b59f6-9rsrc' metadata: creationTimestamp: "2026-06-15T06:09:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:09:59Z" name: isvc-primary-f19eac-predictor-6ff95b59f6.18b92c443d2b2e50 namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: 8fdb3cda-919e-4a78-92bc-f22fbb6158fd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:09:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-f19eac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20276" uid: 0f9cb0c3-b705-4cce-8c7d-45fed578668c kind: Event lastTimestamp: "2026-06-15T06:09:59Z" message: Scaled up replica set isvc-primary-f19eac-predictor-6ff95b59f6 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:09:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:09:59Z" name: isvc-primary-f19eac-predictor.18b92c44393cf4b1 namespace: kserve-ci-e2e-test resourceVersion: "20283" uid: 191e40b4-ac24-4db2-a5a0-21fa41c27e33 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:10:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-f19eac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: b11a3aba-6451-4033-a636-1a2aaa7cfc21 kind: Event lastTimestamp: "2026-06-15T06:10:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:10:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:10:29Z" name: isvc-primary-f19eac-predictor.18b92c47b8019780 namespace: kserve-ci-e2e-test resourceVersion: "20548" uid: 10454af9-a930-425f-ab49-615257991de7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:10:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-f19eac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: b11a3aba-6451-4033-a636-1a2aaa7cfc21 kind: Event lastTimestamp: "2026-06-15T06:10:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:10:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:10:29Z" name: isvc-primary-f19eac-predictor.18b92c47b8648ce7 namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: 811771a8-039f-41a7-accd-06e85046084b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:10:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-f19eac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20444" uid: b11a3aba-6451-4033-a636-1a2aaa7cfc21 kind: Event lastTimestamp: "2026-06-15T06:11:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:10:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:44Z" name: isvc-primary-f19eac-predictor.18b92c4eb5fc92d4 namespace: kserve-ci-e2e-test resourceVersion: "21150" uid: dbe0a21f-4914-4037-b803-c711bb58a450 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:10:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-f19eac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20444" uid: b11a3aba-6451-4033-a636-1a2aaa7cfc21 kind: Event lastTimestamp: "2026-06-15T06:11:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:10:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:44Z" name: isvc-primary-f19eac-predictor.18b92c4eb65a509b namespace: kserve-ci-e2e-test resourceVersion: "21151" uid: 5f88bfff-a2ac-4313-893d-3e960e6baaee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-06-15T06:09:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-f19eac namespace: kserve-ci-e2e-test resourceVersion: "20252" uid: 8526a4c8-f675-4573-8d2a-a08f6b755d34 kind: Event lastTimestamp: "2026-06-15T06:11:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:09:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:23Z" name: isvc-primary-f19eac.18b92c44391295f3 namespace: kserve-ci-e2e-test resourceVersion: "20932" uid: 9af15e4e-6164-4faa-b638-14dd42afd9a5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:09:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-f19eac namespace: kserve-ci-e2e-test resourceVersion: "20252" uid: 8526a4c8-f675-4573-8d2a-a08f6b755d34 kind: Event lastTimestamp: "2026-06-15T06:09:59Z" message: 'Failed to update status for InferenceService "isvc-primary-f19eac": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-f19eac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:09:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:09:59Z" name: isvc-primary-f19eac.18b92c44397db7ab namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: 4e16eb79-6c81-414c-9cbb-f4660d0e5401 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:09:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-f19eac namespace: kserve-ci-e2e-test resourceVersion: "20252" uid: 8526a4c8-f675-4573-8d2a-a08f6b755d34 kind: Event lastTimestamp: "2026-06-15T06:09:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-f19eac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:09:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:09:59Z" name: isvc-primary-f19eac.18b92c44397dfc8f namespace: kserve-ci-e2e-test resourceVersion: "20287" uid: 37bdeba7-474b-4cb8-8212-6d5f740e06ba reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-f19eac namespace: kserve-ci-e2e-test resourceVersion: "20929" uid: 8526a4c8-f675-4573-8d2a-a08f6b755d34 kind: Event lastTimestamp: "2026-06-15T06:11:23Z" message: InferenceService [isvc-primary-f19eac] is Ready metadata: creationTimestamp: "2026-06-15T06:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:23Z" name: isvc-primary-f19eac.18b92c57c9f51f4a namespace: kserve-ci-e2e-test resourceVersion: "20931" uid: 0c6ee937-e7b1-405a-b161-c382e49cfe0d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T05:59:27.858727Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14906" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 to ip-10-0-135-167.ec2.internal metadata: creationTimestamp: "2026-06-15T05:59:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T05:59:27Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bb13e73fc9a namespace: kserve-ci-e2e-test resourceVersion: "14913" uid: 8cd688f2-1c8e-4759-84de-7d302fe64f1b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65bc5545-rprtl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T05:59:28Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T05:59:28Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T05:59:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T05:59:28Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bb158329789 namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: 79e7cc48-6fcb-4e05-834a-d157fd5c87a8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T05:59:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T05:59:28Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ba8edcbfb3f9312d158be16483785d7654e60c7090f262c42214fd2b29effada" already present on machine metadata: creationTimestamp: "2026-06-15T05:59:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T05:59:28Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bb159948202 namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: b73090c6-bcaf-4b0b-8add-573eea19c75f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T05:59:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T05:59:28Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-15T05:59:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T05:59:28Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bb15cb2af68 namespace: kserve-ci-e2e-test resourceVersion: "14931" uid: d5661075-6c5f-4bfe-b5f8-3599dc4b4858 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T05:59:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T05:59:28Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-15T05:59:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T05:59:28Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bb15d2ac0fc namespace: kserve-ci-e2e-test resourceVersion: "14932" uid: fb60c9c6-ce6a-4e4a-b661-8bf66bba25a1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T05:59:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T05:59:32Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1619" metadata: creationTimestamp: "2026-06-15T05:59:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T05:59:32Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bb257af48ae namespace: kserve-ci-e2e-test resourceVersion: "14967" uid: 7554d120-6b50-4a6d-bc9c-2717e1bab085 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T05:59:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T05:59:45Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1619" in 13.303s (13.303s including waiting). Image size: 1555617714 bytes.' metadata: creationTimestamp: "2026-06-15T05:59:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T05:59:45Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bb570aaa025 namespace: kserve-ci-e2e-test resourceVersion: "15064" uid: 4f51dac9-60bd-46f2-b7a3-b6b798600492 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T05:59:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T05:59:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T05:59:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T05:59:45Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bb573e3e796 namespace: kserve-ci-e2e-test resourceVersion: "15066" uid: 80928dde-af4b-459c-b3e9-2f6428eb8951 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T05:59:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T05:59:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T05:59:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T05:59:45Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bb5745ae3f7 namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: b692022d-7c0c-4ffc-a978-c412d452134e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T05:59:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T05:59:45Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-06-15T05:59:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T05:59:45Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bb57468cd08 namespace: kserve-ci-e2e-test resourceVersion: "15068" uid: 9364c1b0-faa6-449f-ba50-e4af6d3e2235 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T05:59:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T05:59:48Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.125s (2.125s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-06-15T05:59:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T05:59:48Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bb5f31d8bdc namespace: kserve-ci-e2e-test resourceVersion: "15085" uid: 8e2b8ec7-f861-467d-979f-3c1653e25381 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T05:59:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T05:59:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T05:59:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T05:59:48Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bb5f5da55e6 namespace: kserve-ci-e2e-test resourceVersion: "15087" uid: 4fd6432d-b1ef-4f86-b80a-4d6fb622b3db reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T05:59:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T05:59:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T05:59:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T05:59:48Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bb5f64e16a5 namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: 06d17759-1435-40a8-88aa-840436ad04c2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T05:59:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T05:59:48Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:41e9c610ce929fc56f4076f97bf42d251460cf8b6a90f49e0a84e5b92946522f" metadata: creationTimestamp: "2026-06-15T05:59:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T05:59:48Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bb5f65c9300 namespace: kserve-ci-e2e-test resourceVersion: "15089" uid: a184ea21-a141-4230-bc62-1ca926ff812c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T05:59:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T05:59:50Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:41e9c610ce929fc56f4076f97bf42d251460cf8b6a90f49e0a84e5b92946522f" in 2.078s (2.078s including waiting). Image size: 239737436 bytes.' metadata: creationTimestamp: "2026-06-15T05:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T05:59:50Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bb6723b5b09 namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: 794468c2-f9e6-4036-af0d-c8ea3f94ef02 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T05:59:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T05:59:50Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-06-15T05:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T05:59:50Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bb6751ac48a namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 97d66e72-4600-41e7-8119-c2686f0aa8d9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T05:59:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T05:59:50Z" message: Started container agent metadata: creationTimestamp: "2026-06-15T05:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T05:59:50Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bb675923aad namespace: kserve-ci-e2e-test resourceVersion: "15106" uid: 1a3b33f9-2f64-4331-9b73-64292868e956 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-06-15T05:59:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T06:01:29Z" message: 'Readiness probe failed: dial tcp 10.132.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T05:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:29Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bb68dd96f3a namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: f2811a78-1739-4bf3-a489-e9a49df3a9ab reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-06-15T05:59:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T06:01:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-06-15T05:59:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:29Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bb6c9a60d95 namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 3ccfc7ba-0c55-49df-aa4f-9c2e693ccc46 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T06:01:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:22Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bcc06320a6c namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: 135eb86d-5bec-4849-a2c4-99fbef02828f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T06:01:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:22Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bcc063329a3 namespace: kserve-ci-e2e-test resourceVersion: "15828" uid: a32589d4-5de1-4f2e-b96b-c70f2c513e91 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T06:01:22Z" message: Stopping container agent metadata: creationTimestamp: "2026-06-15T06:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:22Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bcc0636d1a5 namespace: kserve-ci-e2e-test resourceVersion: "15823" uid: dd34e71f-987a-455c-9ba9-1b466129ab0f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: c31d7dde-0e0f-40c5-b59a-802fb3c3022f kind: Event lastTimestamp: "2026-06-15T06:01:32Z" message: 'Readiness probe failed: Get "https://10.132.0.19:8643/healthz": dial tcp 10.132.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:32Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7.18b92bcd23f0618c namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: 4a077e45-3164-43c6-9182-e95d24dad18c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T05:59:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 6761a202-611e-4301-9d3d-b18b72d08791 kind: Event lastTimestamp: "2026-06-15T05:59:27Z" message: 'Created pod: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c-tcpf7' metadata: creationTimestamp: "2026-06-15T05:59:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T05:59:27Z" name: isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c.18b92bb13dda8f84 namespace: kserve-ci-e2e-test resourceVersion: "14908" uid: 364fdd03-cb0d-47a3-9455-199d7ca291fe reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T05:59:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-cb69f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: 06314887-01d1-493b-be12-832054361a05 kind: Event lastTimestamp: "2026-06-15T05:59:27Z" message: Scaled up replica set isvc-raw-sklearn-batcher-cb69f-predictor-5b69cd895c from 0 to 1 metadata: creationTimestamp: "2026-06-15T05:59:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T05:59:27Z" name: isvc-raw-sklearn-batcher-cb69f-predictor.18b92bb1369acc5f namespace: kserve-ci-e2e-test resourceVersion: "14896" uid: 42d21079-649f-400b-ab66-d00fd889817a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T05:59:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-cb69f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: 6c25afeb-97b9-4e5c-8d92-81d5206ebcf3 kind: Event lastTimestamp: "2026-06-15T06:00:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T05:59:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:00:12Z" name: isvc-raw-sklearn-batcher-cb69f-predictor.18b92bb4bbe9cd85 namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: b743a98a-f61a-495c-86c5-dd32f88e4fbc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T05:59:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-cb69f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: 6c25afeb-97b9-4e5c-8d92-81d5206ebcf3 kind: Event lastTimestamp: "2026-06-15T06:00:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T05:59:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:00:12Z" name: isvc-raw-sklearn-batcher-cb69f-predictor.18b92bb4bc6e727f namespace: kserve-ci-e2e-test resourceVersion: "15271" uid: 4c61ee93-e55a-40e7-bce2-6d9011112afe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:00:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-cb69f-predictor namespace: kserve-ci-e2e-test resourceVersion: "15041" uid: 6c25afeb-97b9-4e5c-8d92-81d5206ebcf3 kind: Event lastTimestamp: "2026-06-15T06:01:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:00:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:01:12Z" name: isvc-raw-sklearn-batcher-cb69f-predictor.18b92bbf38cf9b90 namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: 3ce86f91-e2be-44bb-8ebd-f1c4acb0a84a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:00:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-cb69f-predictor namespace: kserve-ci-e2e-test resourceVersion: "15041" uid: 6c25afeb-97b9-4e5c-8d92-81d5206ebcf3 kind: Event lastTimestamp: "2026-06-15T06:01:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:00:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:01:12Z" name: isvc-raw-sklearn-batcher-cb69f-predictor.18b92bbf39322b5f namespace: kserve-ci-e2e-test resourceVersion: "15720" uid: 58e6bbe2-7106-451d-aa2a-5a8fa47cdaec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-06-15T05:59:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-cb69f namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: 1b7eb606-2a6e-4a16-9a24-fc03648ebc28 kind: Event lastTimestamp: "2026-06-15T06:01:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T05:59:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:01:09Z" name: isvc-raw-sklearn-batcher-cb69f.18b92bb13cade894 namespace: kserve-ci-e2e-test resourceVersion: "15692" uid: c74506ee-bcf5-4af4-92c4-cbd5411cb2f4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T05:59:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-cb69f namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: 1b7eb606-2a6e-4a16-9a24-fc03648ebc28 kind: Event lastTimestamp: "2026-06-15T05:59:27Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-cb69f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-cb69f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T05:59:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T05:59:27Z" name: isvc-raw-sklearn-batcher-cb69f.18b92bb13d139949 namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: a159f4ed-1820-418d-9355-71da61f0d3b2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T05:59:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-cb69f namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: 1b7eb606-2a6e-4a16-9a24-fc03648ebc28 kind: Event lastTimestamp: "2026-06-15T05:59:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-cb69f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T05:59:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T05:59:27Z" name: isvc-raw-sklearn-batcher-cb69f.18b92bb13d13fa60 namespace: kserve-ci-e2e-test resourceVersion: "14922" uid: 754055cb-0c2a-4ae1-8e9d-fb6691803950 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-cb69f namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 1b7eb606-2a6e-4a16-9a24-fc03648ebc28 kind: Event lastTimestamp: "2026-06-15T06:01:09Z" message: InferenceService [isvc-raw-sklearn-batcher-cb69f] is Ready metadata: creationTimestamp: "2026-06-15T06:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:01:09Z" name: isvc-raw-sklearn-batcher-cb69f.18b92bc8da66195a namespace: kserve-ci-e2e-test resourceVersion: "15691" uid: fd662d52-2455-42a2-89cc-1e949167c5fc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:11:29.412877Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-f19eac-predictor-6d97b7b459-hs9ml namespace: kserve-ci-e2e-test resourceVersion: "21003" uid: fb9e63f1-cdeb-4e5b-9952-b48937fb1e74 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-f19eac-predictor-6d97b7b459-hs9ml to ip-10-0-135-167.ec2.internal metadata: creationTimestamp: "2026-06-15T06:11:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:11:29Z" name: isvc-secondary-f19eac-predictor-6d97b7b459-hs9ml.18b92c593e6e904b namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: ae4d2a97-6ea0-4b2f-a5be-bb1f93e53934 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65bc5545-rprtl source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:11:29Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-f19eac-predictor-6d97b7b459-hs9ml namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: fb9e63f1-cdeb-4e5b-9952-b48937fb1e74 kind: Event lastTimestamp: "2026-06-15T06:11:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-secondary-f19eac-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:11:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:30Z" name: isvc-secondary-f19eac-predictor-6d97b7b459-hs9ml.18b92c5944f98c4d namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: ccdfb3f7-80e7-4ffa-8846-2e8eed39718c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-f19eac-predictor-6d97b7b459-hs9ml namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: fb9e63f1-cdeb-4e5b-9952-b48937fb1e74 kind: Event lastTimestamp: "2026-06-15T06:11:31Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:11:31Z" name: isvc-secondary-f19eac-predictor-6d97b7b459-hs9ml.18b92c59b16daacf namespace: kserve-ci-e2e-test resourceVersion: "21037" uid: ae08d202-c896-454c-8f2b-c9b8b18d646d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:11:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-f19eac-predictor-6d97b7b459-hs9ml namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: fb9e63f1-cdeb-4e5b-9952-b48937fb1e74 kind: Event lastTimestamp: "2026-06-15T06:11:35Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ba8edcbfb3f9312d158be16483785d7654e60c7090f262c42214fd2b29effada" already present on machine metadata: creationTimestamp: "2026-06-15T06:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:35Z" name: isvc-secondary-f19eac-predictor-6d97b7b459-hs9ml.18b92c59b2ce4cb0 namespace: kserve-ci-e2e-test resourceVersion: "21075" uid: a2b3c986-0660-4a05-97ec-190e0f866b95 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:11:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-f19eac-predictor-6d97b7b459-hs9ml namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: fb9e63f1-cdeb-4e5b-9952-b48937fb1e74 kind: Event lastTimestamp: "2026-06-15T06:11:35Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-15T06:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:35Z" name: isvc-secondary-f19eac-predictor-6d97b7b459-hs9ml.18b92c59b6872be2 namespace: kserve-ci-e2e-test resourceVersion: "21079" uid: ae392210-68be-4ef5-8f45-15551f3d3ee9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:11:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-f19eac-predictor-6d97b7b459-hs9ml namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: fb9e63f1-cdeb-4e5b-9952-b48937fb1e74 kind: Event lastTimestamp: "2026-06-15T06:11:35Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-15T06:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:35Z" name: isvc-secondary-f19eac-predictor-6d97b7b459-hs9ml.18b92c59b7167dab namespace: kserve-ci-e2e-test resourceVersion: "21080" uid: 95f28f37-372d-45a1-b4f4-38b153f71a3a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-f19eac-predictor-6d97b7b459-hs9ml namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: fb9e63f1-cdeb-4e5b-9952-b48937fb1e74 kind: Event lastTimestamp: "2026-06-15T06:11:42Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-f19eac-predictor-6d97b7b459-hs9ml_kserve-ci-e2e-test(fb9e63f1-cdeb-4e5b-9952-b48937fb1e74) metadata: creationTimestamp: "2026-06-15T06:11:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:42Z" name: isvc-secondary-f19eac-predictor-6d97b7b459-hs9ml.18b92c5c44e5e5de namespace: kserve-ci-e2e-test resourceVersion: "21131" uid: 10858efd-6b50-48ca-8355-dd6ba48fa51d reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-f19eac-predictor-6d97b7b459 namespace: kserve-ci-e2e-test resourceVersion: "20994" uid: bb3e589a-8760-4e5b-b957-13f8006e9503 kind: Event lastTimestamp: "2026-06-15T06:11:29Z" message: 'Created pod: isvc-secondary-f19eac-predictor-6d97b7b459-hs9ml' metadata: creationTimestamp: "2026-06-15T06:11:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:29Z" name: isvc-secondary-f19eac-predictor-6d97b7b459.18b92c593e06ec6b namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: eefab23d-4d92-450f-84c4-ff23384cdfe1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-f19eac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20992" uid: e181c9c6-8e5c-4521-9e0d-12a8f0b74513 kind: Event lastTimestamp: "2026-06-15T06:11:29Z" message: Scaled up replica set isvc-secondary-f19eac-predictor-6d97b7b459 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:11:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:29Z" name: isvc-secondary-f19eac-predictor.18b92c5939fb0dfc namespace: kserve-ci-e2e-test resourceVersion: "20996" uid: c36c1d7e-1d09-45d4-aa98-7f2eadf4063b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-f19eac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20993" uid: b4700eab-cee3-4fe6-86dd-eb0829aec759 kind: Event lastTimestamp: "2026-06-15T06:11:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:44Z" name: isvc-secondary-f19eac-predictor.18b92c5cb868e640 namespace: kserve-ci-e2e-test resourceVersion: "21153" uid: 099797bb-052b-47e6-8cb9-dd27cf46bac1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-f19eac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20993" uid: b4700eab-cee3-4fe6-86dd-eb0829aec759 kind: Event lastTimestamp: "2026-06-15T06:11:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:44Z" name: isvc-secondary-f19eac-predictor.18b92c5cb8ceb797 namespace: kserve-ci-e2e-test resourceVersion: "21155" uid: b6dba94a-0f0f-4c8c-b1fe-7ecb99db0ece reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-06-15T06:11:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-f19eac namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: 8983b2bf-46e9-441f-99b1-d2927d9a3387 kind: Event lastTimestamp: "2026-06-15T06:11:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:11:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:47Z" name: isvc-secondary-f19eac.18b92c5939d3aacb namespace: kserve-ci-e2e-test resourceVersion: "21169" uid: 8e8c6783-43bf-4b2c-82bf-d0dbfbb36bf0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:11:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-f19eac namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: 8983b2bf-46e9-441f-99b1-d2927d9a3387 kind: Event lastTimestamp: "2026-06-15T06:11:29Z" message: 'Failed to update status for InferenceService "isvc-secondary-f19eac": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-f19eac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:11:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:29Z" name: isvc-secondary-f19eac.18b92c593a2a7caf namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: e8d305d5-eb76-4fd4-93ea-3641af2b9e92 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:11:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-f19eac namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: 8983b2bf-46e9-441f-99b1-d2927d9a3387 kind: Event lastTimestamp: "2026-06-15T06:11:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-f19eac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:11:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:29Z" name: isvc-secondary-f19eac.18b92c593a2ad9f5 namespace: kserve-ci-e2e-test resourceVersion: "21015" uid: 33d2b6f8-610f-480b-a54f-7ac9b5cfe340 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:01:22.966871Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: f66f5b22-5b10-4b20-9c63-d10c7e361d16 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f to ip-10-0-135-167.ec2.internal metadata: creationTimestamp: "2026-06-15T06:01:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:01:22Z" name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f.18b92bcc0b6ee2d1 namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 07aaf263-4e9b-47fe-ba25-b9ea563e83fc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65bc5545-rprtl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:23Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: f66f5b22-5b10-4b20-9c63-d10c7e361d16 kind: Event lastTimestamp: "2026-06-15T06:01:23Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-2af6c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:23Z" name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f.18b92bcc1a1fb92d namespace: kserve-ci-e2e-test resourceVersion: "15879" uid: 20537b1f-4761-409b-889c-3ada74a70f0a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:23Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f namespace: kserve-ci-e2e-test resourceVersion: "15850" uid: f66f5b22-5b10-4b20-9c63-d10c7e361d16 kind: Event lastTimestamp: "2026-06-15T06:01:23Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:01:23Z" name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f.18b92bcc48e72fb4 namespace: kserve-ci-e2e-test resourceVersion: "15895" uid: 153a4d10-9e31-4f7f-8f21-2b01239224a9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: f66f5b22-5b10-4b20-9c63-d10c7e361d16 kind: Event lastTimestamp: "2026-06-15T06:01:24Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ba8edcbfb3f9312d158be16483785d7654e60c7090f262c42214fd2b29effada" already present on machine metadata: creationTimestamp: "2026-06-15T06:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:24Z" name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f.18b92bcc4a54889a namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: 19b16af8-3bfe-43d8-b223-852e31455665 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: f66f5b22-5b10-4b20-9c63-d10c7e361d16 kind: Event lastTimestamp: "2026-06-15T06:01:24Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-15T06:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:24Z" name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f.18b92bcc4d5c8967 namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: d3e2e07c-7174-4458-b064-998fce20afc8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: f66f5b22-5b10-4b20-9c63-d10c7e361d16 kind: Event lastTimestamp: "2026-06-15T06:01:24Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-15T06:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:24Z" name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f.18b92bcc4dede277 namespace: kserve-ci-e2e-test resourceVersion: "15905" uid: bcd0f47c-909e-404c-a9e9-625de41afe7b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: f66f5b22-5b10-4b20-9c63-d10c7e361d16 kind: Event lastTimestamp: "2026-06-15T06:01:28Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:01:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:28Z" name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f.18b92bcd39147f73 namespace: kserve-ci-e2e-test resourceVersion: "15942" uid: 89eeb240-7e86-4ee9-beb0-1a21ac82c0ac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: f66f5b22-5b10-4b20-9c63-d10c7e361d16 kind: Event lastTimestamp: "2026-06-15T06:01:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:01:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:28Z" name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f.18b92bcd3d46bc6c namespace: kserve-ci-e2e-test resourceVersion: "15949" uid: 83630af3-7628-458d-a277-34d22d8c6a85 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: f66f5b22-5b10-4b20-9c63-d10c7e361d16 kind: Event lastTimestamp: "2026-06-15T06:01:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:01:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:28Z" name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f.18b92bcd3dc597c8 namespace: kserve-ci-e2e-test resourceVersion: "15950" uid: bd501564-fc47-4aaf-8749-821a63c91da8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: f66f5b22-5b10-4b20-9c63-d10c7e361d16 kind: Event lastTimestamp: "2026-06-15T06:01:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:01:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:28Z" name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f.18b92bcd3dd23db7 namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: ba34160b-87da-4dba-ae6a-df70bdd77a53 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: f66f5b22-5b10-4b20-9c63-d10c7e361d16 kind: Event lastTimestamp: "2026-06-15T06:01:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:01:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:28Z" name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f.18b92bcd41631c93 namespace: kserve-ci-e2e-test resourceVersion: "15955" uid: 551a6d0b-0459-465f-bf40-86d73f97aabd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: f66f5b22-5b10-4b20-9c63-d10c7e361d16 kind: Event lastTimestamp: "2026-06-15T06:01:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:01:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:28Z" name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f.18b92bcd41e01e64 namespace: kserve-ci-e2e-test resourceVersion: "15956" uid: 4c899687-2448-4392-92eb-fe19c414d0ec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-06-15T06:01:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: f66f5b22-5b10-4b20-9c63-d10c7e361d16 kind: Event lastTimestamp: "2026-06-15T06:03:25Z" message: 'Readiness probe failed: dial tcp 10.132.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:01:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:25Z" name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f.18b92bcd7576a861 namespace: kserve-ci-e2e-test resourceVersion: "17027" uid: 0d81a6ab-7f5b-440f-9a7f-8e1e5b3a4a19 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: f66f5b22-5b10-4b20-9c63-d10c7e361d16 kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:23Z" name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f.18b92be80d69b24f namespace: kserve-ci-e2e-test resourceVersion: "16924" uid: a8770e2f-6b36-4919-9488-40f58f7929e7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: f66f5b22-5b10-4b20-9c63-d10c7e361d16 kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:23Z" name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f.18b92be80d6a545b namespace: kserve-ci-e2e-test resourceVersion: "16925" uid: ee003e96-5a18-4ae4-85dc-34c8c5080574 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: f66f5b22-5b10-4b20-9c63-d10c7e361d16 kind: Event lastTimestamp: "2026-06-15T06:03:25Z" message: 'Readiness probe failed: Get "https://10.132.0.20:8643/healthz": dial tcp 10.132.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:25Z" name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f.18b92be877d2bd49 namespace: kserve-ci-e2e-test resourceVersion: "17026" uid: b1fa34fe-31bb-46c7-a70e-e3fbcb41d573 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545 namespace: kserve-ci-e2e-test resourceVersion: "15822" uid: 29334e64-aaa5-4003-bf9e-2a6d2e4a27d3 kind: Event lastTimestamp: "2026-06-15T06:01:22Z" message: 'Created pod: isvc-sklearn-graph-raw-2af6c-predictor-74db776545-ldr9f' metadata: creationTimestamp: "2026-06-15T06:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:01:22Z" name: isvc-sklearn-graph-raw-2af6c-predictor-74db776545.18b92bcc0b022bcc namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: 1ed871d9-125b-4aba-821a-4abe1b288a87 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-2af6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: 085e28bc-7c2b-4634-9bb8-db55c5014a1e kind: Event lastTimestamp: "2026-06-15T06:01:22Z" message: Scaled up replica set isvc-sklearn-graph-raw-2af6c-predictor-74db776545 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:01:22Z" name: isvc-sklearn-graph-raw-2af6c-predictor.18b92bcc06db1b97 namespace: kserve-ci-e2e-test resourceVersion: "15824" uid: b09d469e-778f-4def-a42c-85cf7d0f616a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:01:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-2af6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15821" uid: f1ccbe85-b391-41cc-b539-21a603c7a44c kind: Event lastTimestamp: "2026-06-15T06:02:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:01:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:22Z" name: isvc-sklearn-graph-raw-2af6c-predictor.18b92bcf85cfd8d4 namespace: kserve-ci-e2e-test resourceVersion: "16375" uid: aeb6f47c-9817-474d-826b-c4b0c93fdf97 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:01:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-2af6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15821" uid: f1ccbe85-b391-41cc-b539-21a603c7a44c kind: Event lastTimestamp: "2026-06-15T06:02:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:01:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:22Z" name: isvc-sklearn-graph-raw-2af6c-predictor.18b92bcf8635ab76 namespace: kserve-ci-e2e-test resourceVersion: "16376" uid: 3a7a54ee-c4cc-446f-823d-fdb85a5d0284 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:02:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-2af6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: f1ccbe85-b391-41cc-b539-21a603c7a44c kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:23Z" name: isvc-sklearn-graph-raw-2af6c-predictor.18b92bdd824dcbf0 namespace: kserve-ci-e2e-test resourceVersion: "16865" uid: 9f34427e-43d7-4a9c-b1a4-bb62cd9632cf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:02:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-2af6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: f1ccbe85-b391-41cc-b539-21a603c7a44c kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:23Z" name: isvc-sklearn-graph-raw-2af6c-predictor.18b92bdd8315ba20 namespace: kserve-ci-e2e-test resourceVersion: "16868" uid: f9d9c6ff-e1a5-4f38-85b7-13183a1b5606 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-06-15T06:01:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-2af6c namespace: kserve-ci-e2e-test resourceVersion: "15795" uid: 506e65f1-4ebf-48a6-9810-81bc30b6d478 kind: Event lastTimestamp: "2026-06-15T06:02:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:02:55Z" name: isvc-sklearn-graph-raw-2af6c.18b92bcc06e25e15 namespace: kserve-ci-e2e-test resourceVersion: "16623" uid: e7353bd5-1b8a-4f55-953e-8e91f2cd82d0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-2af6c namespace: kserve-ci-e2e-test resourceVersion: "15795" uid: 506e65f1-4ebf-48a6-9810-81bc30b6d478 kind: Event lastTimestamp: "2026-06-15T06:01:22Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-2af6c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-2af6c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:01:22Z" name: isvc-sklearn-graph-raw-2af6c.18b92bcc07692ed0 namespace: kserve-ci-e2e-test resourceVersion: "15829" uid: b1ee932a-0ae4-4fb6-9635-84fbe3075ab5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-2af6c namespace: kserve-ci-e2e-test resourceVersion: "16621" uid: 506e65f1-4ebf-48a6-9810-81bc30b6d478 kind: Event lastTimestamp: "2026-06-15T06:02:55Z" message: InferenceService [isvc-sklearn-graph-raw-2af6c] is Ready metadata: creationTimestamp: "2026-06-15T06:02:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:02:55Z" name: isvc-sklearn-graph-raw-2af6c.18b92be1812f75fe namespace: kserve-ci-e2e-test resourceVersion: "16622" uid: 2045be97-ddcd-4626-b3a4-aa7c971efd65 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:03:23.313224Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8 namespace: kserve-ci-e2e-test resourceVersion: "16934" uid: f1c43bf0-8ba9-49d7-8dc9-af68f3a2c07d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8 to ip-10-0-135-167.ec2.internal metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:03:23Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8.18b92be810a277f5 namespace: kserve-ci-e2e-test resourceVersion: "16943" uid: f84557b3-fef7-4754-9e92-97bc6a612a6e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65bc5545-rprtl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:23Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8 namespace: kserve-ci-e2e-test resourceVersion: "16953" uid: f1c43bf0-8ba9-49d7-8dc9-af68f3a2c07d kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:03:23Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8.18b92be82a5fec91 namespace: kserve-ci-e2e-test resourceVersion: "16993" uid: fc96b2e0-ae38-4fdb-a851-1dadf987af5d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8 namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: f1c43bf0-8ba9-49d7-8dc9-af68f3a2c07d kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ba8edcbfb3f9312d158be16483785d7654e60c7090f262c42214fd2b29effada" already present on machine metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:23Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8.18b92be82ba07e2e namespace: kserve-ci-e2e-test resourceVersion: "16995" uid: 11c61d4b-7bc4-4d5a-bcbd-301ffd563ef1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8 namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: f1c43bf0-8ba9-49d7-8dc9-af68f3a2c07d kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:23Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8.18b92be82ea3a730 namespace: kserve-ci-e2e-test resourceVersion: "16997" uid: e20d876f-5a67-4432-93fb-3333658ed695 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8 namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: f1c43bf0-8ba9-49d7-8dc9-af68f3a2c07d kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:23Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8.18b92be82f7cc2c3 namespace: kserve-ci-e2e-test resourceVersion: "16998" uid: acfe4fcf-2cec-4a04-96aa-450b6763253c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8 namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: f1c43bf0-8ba9-49d7-8dc9-af68f3a2c07d kind: Event lastTimestamp: "2026-06-15T06:03:28Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:28Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8.18b92be946b94e20 namespace: kserve-ci-e2e-test resourceVersion: "17060" uid: 03f44129-ded0-4625-ae43-4b6c917d4224 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8 namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: f1c43bf0-8ba9-49d7-8dc9-af68f3a2c07d kind: Event lastTimestamp: "2026-06-15T06:03:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:28Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8.18b92be94a248497 namespace: kserve-ci-e2e-test resourceVersion: "17068" uid: 573a5159-42e8-4847-ae19-f6753c8c779a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8 namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: f1c43bf0-8ba9-49d7-8dc9-af68f3a2c07d kind: Event lastTimestamp: "2026-06-15T06:03:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:28Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8.18b92be94a9f7533 namespace: kserve-ci-e2e-test resourceVersion: "17070" uid: 6a714cc3-ad23-4844-b45d-75e6f009ecb8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8 namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: f1c43bf0-8ba9-49d7-8dc9-af68f3a2c07d kind: Event lastTimestamp: "2026-06-15T06:03:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:28Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8.18b92be94aab4c80 namespace: kserve-ci-e2e-test resourceVersion: "17071" uid: 304c54d0-0eb2-46a8-80e7-402488a03239 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8 namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: f1c43bf0-8ba9-49d7-8dc9-af68f3a2c07d kind: Event lastTimestamp: "2026-06-15T06:03:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:28Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8.18b92be94e1f844d namespace: kserve-ci-e2e-test resourceVersion: "17077" uid: a1c5bc45-9ab3-44a0-a520-620caae43a94 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8 namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: f1c43bf0-8ba9-49d7-8dc9-af68f3a2c07d kind: Event lastTimestamp: "2026-06-15T06:03:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:28Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8.18b92be94ea5d6ff namespace: kserve-ci-e2e-test resourceVersion: "17079" uid: 1ba1d078-d3f8-4dec-a159-61755025f50d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-06-15T06:03:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8 namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: f1c43bf0-8ba9-49d7-8dc9-af68f3a2c07d kind: Event lastTimestamp: "2026-06-15T06:05:16Z" message: 'Readiness probe failed: dial tcp 10.132.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:03:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:16Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8.18b92be9bf1cf29d namespace: kserve-ci-e2e-test resourceVersion: "18034" uid: bba9a849-3968-4bc9-addb-17bd1096590c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8 namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: f1c43bf0-8ba9-49d7-8dc9-af68f3a2c07d kind: Event lastTimestamp: "2026-06-15T06:05:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:13Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8.18b92c01c0edb9cf namespace: kserve-ci-e2e-test resourceVersion: "17987" uid: 15ab81b2-ff78-4cdc-8d8c-b736fe6d2caf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8 namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: f1c43bf0-8ba9-49d7-8dc9-af68f3a2c07d kind: Event lastTimestamp: "2026-06-15T06:05:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:13Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8.18b92c01c10196a2 namespace: kserve-ci-e2e-test resourceVersion: "17989" uid: 885490be-74b5-427a-bfec-6e7795750287 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8 namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: f1c43bf0-8ba9-49d7-8dc9-af68f3a2c07d kind: Event lastTimestamp: "2026-06-15T06:05:16Z" message: 'Readiness probe failed: Get "https://10.132.0.22:8643/healthz": dial tcp 10.132.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:05:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:16Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8.18b92c026d66376a namespace: kserve-ci-e2e-test resourceVersion: "18033" uid: 55f84525-3b72-467d-a6d4-b66098909193 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697 namespace: kserve-ci-e2e-test resourceVersion: "16904" uid: 228c7598-661c-4c1e-af47-9bd7f8414b5d kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697-2wbg8' metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:23Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697.18b92be81012abb5 namespace: kserve-ci-e2e-test resourceVersion: "16938" uid: 6c434130-db5e-4190-ba13-9dca6768cc39 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "16901" uid: fd7c7d0f-332d-4486-9399-f143638bde90 kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-ea1cf-predictor-746cbfc697 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:23Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor.18b92be80bb1d8f6 namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: 93c69a38-18c2-4ec1-b32d-69deee212825 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:03:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "16903" uid: beb34a81-ab9e-4627-8f92-23a86e31cc40 kind: Event lastTimestamp: "2026-06-15T06:04:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:03:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:04:08Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor.18b92beb8a74e642 namespace: kserve-ci-e2e-test resourceVersion: "17382" uid: a8136c64-806d-440c-8498-062c1f450ced reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:03:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "16903" uid: beb34a81-ab9e-4627-8f92-23a86e31cc40 kind: Event lastTimestamp: "2026-06-15T06:04:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:03:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:04:08Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor.18b92beb8ad4ea18 namespace: kserve-ci-e2e-test resourceVersion: "17383" uid: 05625b8f-c62b-40bc-8f07-ae6148cb0266 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:04:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "17159" uid: beb34a81-ab9e-4627-8f92-23a86e31cc40 kind: Event lastTimestamp: "2026-06-15T06:05:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:04:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:05:08Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor.18b92bf60796731e namespace: kserve-ci-e2e-test resourceVersion: "17865" uid: e7f26e4c-adda-4799-9cc9-f17a93a3a3dd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:04:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "17159" uid: beb34a81-ab9e-4627-8f92-23a86e31cc40 kind: Event lastTimestamp: "2026-06-15T06:05:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:04:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:05:08Z" name: isvc-sklearn-graph-raw-hpa-ea1cf-predictor.18b92bf607d8f40b namespace: kserve-ci-e2e-test resourceVersion: "17866" uid: 5ce7638c-fa87-45c8-9a3d-608577af2a09 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-06-15T06:03:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-ea1cf namespace: kserve-ci-e2e-test resourceVersion: "16893" uid: b05316e2-13d7-4d14-8040-f14d5bd2c523 kind: Event lastTimestamp: "2026-06-15T06:04:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:04:46Z" name: isvc-sklearn-graph-raw-hpa-ea1cf.18b92be80b8eb70f namespace: kserve-ci-e2e-test resourceVersion: "17679" uid: 90e19b33-da90-45fd-a587-90a95dbe1ec0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:04:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-ea1cf namespace: kserve-ci-e2e-test resourceVersion: "17676" uid: b05316e2-13d7-4d14-8040-f14d5bd2c523 kind: Event lastTimestamp: "2026-06-15T06:04:46Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-ea1cf] is Ready metadata: creationTimestamp: "2026-06-15T06:04:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:04:46Z" name: isvc-sklearn-graph-raw-hpa-ea1cf.18b92bfb76961cf0 namespace: kserve-ci-e2e-test resourceVersion: "17677" uid: 9cf4c01b-503c-4a52-b9a1-9bf47d67d5c0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:07:08.933679Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb namespace: kserve-ci-e2e-test resourceVersion: "18984" uid: f5f10b72-a7b4-40aa-bed4-8e3b2c06ee12 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb to ip-10-0-135-167.ec2.internal metadata: creationTimestamp: "2026-06-15T06:07:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:07:08Z" name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb.18b92c1c98a963d2 namespace: kserve-ci-e2e-test resourceVersion: "18989" uid: 7ce4981a-fca4-49cb-80a8-673473b8b1f8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65bc5545-rprtl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:07:09Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb namespace: kserve-ci-e2e-test resourceVersion: "19005" uid: f5f10b72-a7b4-40aa-bed4-8e3b2c06ee12 kind: Event lastTimestamp: "2026-06-15T06:07:09Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:07:09Z" name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb.18b92c1cb2948b5d namespace: kserve-ci-e2e-test resourceVersion: "19018" uid: 70d92eb7-f1fc-4db5-ac8f-5167d58d30c9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: f5f10b72-a7b4-40aa-bed4-8e3b2c06ee12 kind: Event lastTimestamp: "2026-06-15T06:07:09Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ba8edcbfb3f9312d158be16483785d7654e60c7090f262c42214fd2b29effada" already present on machine metadata: creationTimestamp: "2026-06-15T06:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:07:09Z" name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb.18b92c1cb40abc2d namespace: kserve-ci-e2e-test resourceVersion: "19020" uid: da97deb8-7296-4ef2-a56b-d19eb31d56e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: f5f10b72-a7b4-40aa-bed4-8e3b2c06ee12 kind: Event lastTimestamp: "2026-06-15T06:07:09Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-15T06:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:07:09Z" name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb.18b92c1cb70e2e18 namespace: kserve-ci-e2e-test resourceVersion: "19022" uid: 399e7fe1-9c65-46e5-a41c-089f1f11ad1c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: f5f10b72-a7b4-40aa-bed4-8e3b2c06ee12 kind: Event lastTimestamp: "2026-06-15T06:07:09Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-15T06:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:07:09Z" name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb.18b92c1cb783aecc namespace: kserve-ci-e2e-test resourceVersion: "19023" uid: e84a6d3d-ec5f-4ed7-9bca-ced813c46ebb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: f5f10b72-a7b4-40aa-bed4-8e3b2c06ee12 kind: Event lastTimestamp: "2026-06-15T06:07:13Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:07:13Z" name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb.18b92c1da29c5c2f namespace: kserve-ci-e2e-test resourceVersion: "19062" uid: bca1b7c5-4d99-4ac8-bede-73d5d265fd7c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: f5f10b72-a7b4-40aa-bed4-8e3b2c06ee12 kind: Event lastTimestamp: "2026-06-15T06:07:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:07:13Z" name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb.18b92c1da5d08439 namespace: kserve-ci-e2e-test resourceVersion: "19064" uid: cfcb50ed-1e8e-4110-9a04-8c1b001baf06 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: f5f10b72-a7b4-40aa-bed4-8e3b2c06ee12 kind: Event lastTimestamp: "2026-06-15T06:07:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:07:13Z" name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb.18b92c1da64583f6 namespace: kserve-ci-e2e-test resourceVersion: "19067" uid: b9cbad5e-d3ff-418a-a84a-aa6ad66d27bc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: f5f10b72-a7b4-40aa-bed4-8e3b2c06ee12 kind: Event lastTimestamp: "2026-06-15T06:07:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:07:13Z" name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb.18b92c1da6502d73 namespace: kserve-ci-e2e-test resourceVersion: "19068" uid: 92c6e8db-c155-4486-9145-197b421f6e06 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: f5f10b72-a7b4-40aa-bed4-8e3b2c06ee12 kind: Event lastTimestamp: "2026-06-15T06:07:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:07:13Z" name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb.18b92c1da9301717 namespace: kserve-ci-e2e-test resourceVersion: "19070" uid: e527c0e6-8b1a-4f1e-a7d9-96507bdf6c00 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: f5f10b72-a7b4-40aa-bed4-8e3b2c06ee12 kind: Event lastTimestamp: "2026-06-15T06:07:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:07:13Z" name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb.18b92c1da9a5af35 namespace: kserve-ci-e2e-test resourceVersion: "19071" uid: e0e7d168-8d04-4877-9724-98844a14b16f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 18 eventTime: null firstTimestamp: "2026-06-15T06:07:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: f5f10b72-a7b4-40aa-bed4-8e3b2c06ee12 kind: Event lastTimestamp: "2026-06-15T06:10:00Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:10:00Z" name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb.18b92c1e1a8af1d8 namespace: kserve-ci-e2e-test resourceVersion: "20315" uid: a8443151-de31-4841-8590-8f0edcf6176f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:09:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: f5f10b72-a7b4-40aa-bed4-8e3b2c06ee12 kind: Event lastTimestamp: "2026-06-15T06:09:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:09:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:09:59Z" name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb.18b92c443801db8d namespace: kserve-ci-e2e-test resourceVersion: "20275" uid: 41458718-3df5-4467-a52f-e8710b542dbc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:09:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: f5f10b72-a7b4-40aa-bed4-8e3b2c06ee12 kind: Event lastTimestamp: "2026-06-15T06:09:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:09:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:09:59Z" name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb.18b92c44380284de namespace: kserve-ci-e2e-test resourceVersion: "20277" uid: 53537556-2709-4214-887d-53a19971e7bc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:10:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: f5f10b72-a7b4-40aa-bed4-8e3b2c06ee12 kind: Event lastTimestamp: "2026-06-15T06:10:06Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:10:06Z" name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb.18b92c44c11e1bf4 namespace: kserve-ci-e2e-test resourceVersion: "20378" uid: 836b94bf-0991-4a84-9d7e-498ea76c6bbe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:07:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c namespace: kserve-ci-e2e-test resourceVersion: "18964" uid: 04ea8488-8535-4c56-8e3a-efacfc8dc867 kind: Event lastTimestamp: "2026-06-15T06:07:08Z" message: 'Created pod: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c-fqmlb' metadata: creationTimestamp: "2026-06-15T06:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:07:08Z" name: isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c.18b92c1c983e8534 namespace: kserve-ci-e2e-test resourceVersion: "18985" uid: 6a3fe255-08d2-4217-bd1b-e000f6a5a2bf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:07:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-5802e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18956" uid: 20fd6ba5-0851-49ee-bc49-cc7aab58d6e1 kind: Event lastTimestamp: "2026-06-15T06:07:08Z" message: Scaled up replica set isvc-sklearn-scale-raw-5802e-predictor-5589cdf8c from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:07:08Z" name: isvc-sklearn-scale-raw-5802e-predictor.18b92c1c93a10878 namespace: kserve-ci-e2e-test resourceVersion: "18967" uid: 23e74b86-868f-4355-b67a-575d572b770e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:07:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-5802e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18963" uid: c99b746a-e6c1-4d72-8ca9-189cb6baca5b kind: Event lastTimestamp: "2026-06-15T06:08:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:07:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:08:08Z" name: isvc-sklearn-scale-raw-5802e-predictor.18b92c2012046578 namespace: kserve-ci-e2e-test resourceVersion: "19472" uid: bcc103b4-ad44-40e9-b277-43f3cd8e6a42 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:07:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-5802e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18963" uid: c99b746a-e6c1-4d72-8ca9-189cb6baca5b kind: Event lastTimestamp: "2026-06-15T06:08:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:07:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:08:08Z" name: isvc-sklearn-scale-raw-5802e-predictor.18b92c20126829d5 namespace: kserve-ci-e2e-test resourceVersion: "19473" uid: 8c75ffa5-d856-4618-9935-e4ab6ae1ce93 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-06-15T06:08:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-5802e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19151" uid: c99b746a-e6c1-4d72-8ca9-189cb6baca5b kind: Event lastTimestamp: "2026-06-15T06:09:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:08:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:09:54Z" name: isvc-sklearn-scale-raw-5802e-predictor.18b92c2e0dc1ee67 namespace: kserve-ci-e2e-test resourceVersion: "20217" uid: 4e186b61-9912-46f6-bc40-cebb5533a150 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-06-15T06:08:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-5802e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19151" uid: c99b746a-e6c1-4d72-8ca9-189cb6baca5b kind: Event lastTimestamp: "2026-06-15T06:09:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:08:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:09:54Z" name: isvc-sklearn-scale-raw-5802e-predictor.18b92c2e0e23567e namespace: kserve-ci-e2e-test resourceVersion: "20218" uid: 682f4890-9d3f-4285-97ac-830ba9a4480c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-06-15T06:07:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-5802e namespace: kserve-ci-e2e-test resourceVersion: "18943" uid: 1ff95d5a-65e9-4315-8d40-480ef6af660e kind: Event lastTimestamp: "2026-06-15T06:09:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:09:50Z" name: isvc-sklearn-scale-raw-5802e.18b92c1c9375af62 namespace: kserve-ci-e2e-test resourceVersion: "20189" uid: 584b836f-053d-45d0-b7ae-708035c3fe2d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:07:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-5802e namespace: kserve-ci-e2e-test resourceVersion: "18943" uid: 1ff95d5a-65e9-4315-8d40-480ef6af660e kind: Event lastTimestamp: "2026-06-15T06:07:08Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-5802e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-5802e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:07:08Z" name: isvc-sklearn-scale-raw-5802e.18b92c1c93ccedae namespace: kserve-ci-e2e-test resourceVersion: "19008" uid: c53ce9b0-c33b-4dc0-aeae-28ca721dc9f3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:07:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-5802e namespace: kserve-ci-e2e-test resourceVersion: "18958" uid: 1ff95d5a-65e9-4315-8d40-480ef6af660e kind: Event lastTimestamp: "2026-06-15T06:07:08Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-5802e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:07:08Z" name: isvc-sklearn-scale-raw-5802e.18b92c1c9a577774 namespace: kserve-ci-e2e-test resourceVersion: "19010" uid: 95a34a92-9fa1-4d12-b97d-3305142d6df5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:09:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-5802e namespace: kserve-ci-e2e-test resourceVersion: "20186" uid: 1ff95d5a-65e9-4315-8d40-480ef6af660e kind: Event lastTimestamp: "2026-06-15T06:09:50Z" message: InferenceService [isvc-sklearn-scale-raw-5802e] is Ready metadata: creationTimestamp: "2026-06-15T06:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:09:50Z" name: isvc-sklearn-scale-raw-5802e.18b92c422781b403 namespace: kserve-ci-e2e-test resourceVersion: "20187" uid: ab6dccc4-37c1-4ae4-94d7-11f1e72ccb4d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:01:23.049976Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: 8e9429c7-456b-4fda-bb71-a3d8fd9d4bfe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh to ip-10-0-135-167.ec2.internal metadata: creationTimestamp: "2026-06-15T06:01:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:01:23Z" name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh.18b92bcc1062fdc3 namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: 20a7de8e-6328-46ad-9710-3f9140b189b2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65bc5545-rprtl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:23Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: 8e9429c7-456b-4fda-bb71-a3d8fd9d4bfe kind: Event lastTimestamp: "2026-06-15T06:01:23Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:01:23Z" name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh.18b92bcc2a080464 namespace: kserve-ci-e2e-test resourceVersion: "15883" uid: c87aef59-6961-4651-b42d-14f3bba5990c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 8e9429c7-456b-4fda-bb71-a3d8fd9d4bfe kind: Event lastTimestamp: "2026-06-15T06:01:23Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ba8edcbfb3f9312d158be16483785d7654e60c7090f262c42214fd2b29effada" already present on machine metadata: creationTimestamp: "2026-06-15T06:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:23Z" name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh.18b92bcc2b4d994b namespace: kserve-ci-e2e-test resourceVersion: "15885" uid: f41daa9d-de29-4cc8-92bb-03edd208aaa5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 8e9429c7-456b-4fda-bb71-a3d8fd9d4bfe kind: Event lastTimestamp: "2026-06-15T06:01:23Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-15T06:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:23Z" name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh.18b92bcc2e617ef7 namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: 053ed25b-21e2-4e77-983d-5c10f897d977 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 8e9429c7-456b-4fda-bb71-a3d8fd9d4bfe kind: Event lastTimestamp: "2026-06-15T06:01:23Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-15T06:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:23Z" name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh.18b92bcc2ed745d2 namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: 2feaeef7-e399-4bc6-8ca2-99f184a5a69a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 8e9429c7-456b-4fda-bb71-a3d8fd9d4bfe kind: Event lastTimestamp: "2026-06-15T06:01:28Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-06-15T06:01:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:28Z" name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh.18b92bcd3956658f namespace: kserve-ci-e2e-test resourceVersion: "15943" uid: a9441215-1a01-466b-83db-bd7ac63c9f34 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 8e9429c7-456b-4fda-bb71-a3d8fd9d4bfe kind: Event lastTimestamp: "2026-06-15T06:01:49Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 21.445s (21.445s including waiting). Image size: 1304984825 bytes.' metadata: creationTimestamp: "2026-06-15T06:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:49Z" name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh.18b92bd237925009 namespace: kserve-ci-e2e-test resourceVersion: "16118" uid: 4e56900b-d4b5-4587-8046-6c5d9b9ea025 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 8e9429c7-456b-4fda-bb71-a3d8fd9d4bfe kind: Event lastTimestamp: "2026-06-15T06:01:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:49Z" name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh.18b92bd23b930267 namespace: kserve-ci-e2e-test resourceVersion: "16120" uid: 60ea75da-a063-42cb-8553-f0adf2069c05 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 8e9429c7-456b-4fda-bb71-a3d8fd9d4bfe kind: Event lastTimestamp: "2026-06-15T06:01:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:49Z" name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh.18b92bd23c17db17 namespace: kserve-ci-e2e-test resourceVersion: "16121" uid: 02649ebe-b66c-4d03-ad04-7ca09bae2a6c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 8e9429c7-456b-4fda-bb71-a3d8fd9d4bfe kind: Event lastTimestamp: "2026-06-15T06:01:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:49Z" name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh.18b92bd23c250188 namespace: kserve-ci-e2e-test resourceVersion: "16122" uid: 362e9547-f48e-419e-9bd0-a98ce18b2948 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 8e9429c7-456b-4fda-bb71-a3d8fd9d4bfe kind: Event lastTimestamp: "2026-06-15T06:01:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:49Z" name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh.18b92bd23f724028 namespace: kserve-ci-e2e-test resourceVersion: "16123" uid: f05be63b-c3e5-4fbb-a601-75d8379d4583 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 8e9429c7-456b-4fda-bb71-a3d8fd9d4bfe kind: Event lastTimestamp: "2026-06-15T06:01:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:49Z" name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh.18b92bd2400530ce namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: b731a9d3-23cb-4e3d-9496-edc824f1e867 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-06-15T06:01:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 8e9429c7-456b-4fda-bb71-a3d8fd9d4bfe kind: Event lastTimestamp: "2026-06-15T06:03:27Z" message: 'Readiness probe failed: dial tcp 10.132.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:27Z" name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh.18b92bd29ab2f9f9 namespace: kserve-ci-e2e-test resourceVersion: "17045" uid: fdab3815-5f7b-421f-9cc8-93dce4ae6a51 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 8e9429c7-456b-4fda-bb71-a3d8fd9d4bfe kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:23Z" name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh.18b92be818a5d622 namespace: kserve-ci-e2e-test resourceVersion: "16977" uid: d243a9a4-50b3-4a5c-99a9-b4cba1181b59 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 8e9429c7-456b-4fda-bb71-a3d8fd9d4bfe kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:23Z" name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh.18b92be818a65ee6 namespace: kserve-ci-e2e-test resourceVersion: "16980" uid: 91c3061e-5007-4bdf-b9bc-7d845ea7a8bd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 8e9429c7-456b-4fda-bb71-a3d8fd9d4bfe kind: Event lastTimestamp: "2026-06-15T06:03:27Z" message: 'Readiness probe failed: Get "https://10.132.0.21:8643/healthz": dial tcp 10.132.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:03:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:27Z" name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh.18b92be8f4eaff0d namespace: kserve-ci-e2e-test resourceVersion: "17044" uid: a85a3acc-e236-4846-bed5-a7027577712b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657 namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: c9d68a3f-6ceb-4f75-bf1f-be1c208628c6 kind: Event lastTimestamp: "2026-06-15T06:01:23Z" message: 'Created pod: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657-v4bqh' metadata: creationTimestamp: "2026-06-15T06:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:01:23Z" name: isvc-xgboost-graph-raw-2af6c-predictor-75f5969657.18b92bcc0fed823a namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: e1073171-daa7-403a-9cbb-01b36905b794 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-2af6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: 89e08040-449d-453e-bca0-b81f8233a206 kind: Event lastTimestamp: "2026-06-15T06:01:22Z" message: Scaled up replica set isvc-xgboost-graph-raw-2af6c-predictor-75f5969657 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:01:22Z" name: isvc-xgboost-graph-raw-2af6c-predictor.18b92bcc0d5aa463 namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 1baba12e-e30d-499c-93fd-16053b3d70e8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:01:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-2af6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: 513cf9ad-c240-457b-b42d-1f26705e825a kind: Event lastTimestamp: "2026-06-15T06:02:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:23Z" name: isvc-xgboost-graph-raw-2af6c-predictor.18b92bcf8bf70e1f namespace: kserve-ci-e2e-test resourceVersion: "16377" uid: ddaab1eb-ed24-4e4f-b933-e215b0feffab reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:01:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-2af6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: 513cf9ad-c240-457b-b42d-1f26705e825a kind: Event lastTimestamp: "2026-06-15T06:02:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:23Z" name: isvc-xgboost-graph-raw-2af6c-predictor.18b92bcf8c5972d7 namespace: kserve-ci-e2e-test resourceVersion: "16378" uid: 78242040-ffdc-4488-9cac-4207e0d12f50 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:02:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-2af6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16038" uid: 513cf9ad-c240-457b-b42d-1f26705e825a kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:23Z" name: isvc-xgboost-graph-raw-2af6c-predictor.18b92bdd870afe6d namespace: kserve-ci-e2e-test resourceVersion: "16876" uid: e9d11305-086d-46d9-b151-2f7690f2e519 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:02:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-2af6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16038" uid: 513cf9ad-c240-457b-b42d-1f26705e825a kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:23Z" name: isvc-xgboost-graph-raw-2af6c-predictor.18b92bdd8778c8f6 namespace: kserve-ci-e2e-test resourceVersion: "16884" uid: 6dd76623-e252-42e5-9bf6-3b47acb46c02 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-06-15T06:01:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-2af6c namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: 11a90122-2dff-46be-8e35-d1f7ebf89db8 kind: Event lastTimestamp: "2026-06-15T06:02:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:02:57Z" name: isvc-xgboost-graph-raw-2af6c.18b92bcc0d62bdd5 namespace: kserve-ci-e2e-test resourceVersion: "16647" uid: 42cdff01-9960-4cf6-afaa-5ad4c2856409 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-2af6c namespace: kserve-ci-e2e-test resourceVersion: "16645" uid: 11a90122-2dff-46be-8e35-d1f7ebf89db8 kind: Event lastTimestamp: "2026-06-15T06:02:57Z" message: InferenceService [isvc-xgboost-graph-raw-2af6c] is Ready metadata: creationTimestamp: "2026-06-15T06:02:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:02:57Z" name: isvc-xgboost-graph-raw-2af6c.18b92be1fdd07920 namespace: kserve-ci-e2e-test resourceVersion: "16646" uid: f84dd87a-9a5d-4e8d-ae2c-f5f09810d85e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:03:23.457011Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq namespace: kserve-ci-e2e-test resourceVersion: "16975" uid: cfeb06b4-e61d-4d7b-89a3-5095f6ac6ebf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq to ip-10-0-135-167.ec2.internal metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:03:23Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq.18b92be819347970 namespace: kserve-ci-e2e-test resourceVersion: "16981" uid: 8acbc616-d86d-491b-ab4f-c9f54e906812 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65bc5545-rprtl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:23Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq namespace: kserve-ci-e2e-test resourceVersion: "16987" uid: cfeb06b4-e61d-4d7b-89a3-5095f6ac6ebf kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:03:23Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq.18b92be833d2d184 namespace: kserve-ci-e2e-test resourceVersion: "17000" uid: f89ff333-718a-4da0-877f-e340bc155d74 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq namespace: kserve-ci-e2e-test resourceVersion: "16978" uid: cfeb06b4-e61d-4d7b-89a3-5095f6ac6ebf kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ba8edcbfb3f9312d158be16483785d7654e60c7090f262c42214fd2b29effada" already present on machine metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:23Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq.18b92be83599c332 namespace: kserve-ci-e2e-test resourceVersion: "17003" uid: 62397156-526f-4a7c-8de0-587ba97297f9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq namespace: kserve-ci-e2e-test resourceVersion: "16978" uid: cfeb06b4-e61d-4d7b-89a3-5095f6ac6ebf kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:23Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq.18b92be838e2d6a4 namespace: kserve-ci-e2e-test resourceVersion: "17006" uid: a5fb3ef3-772c-4c5c-9793-0f5d58bb9fe8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq namespace: kserve-ci-e2e-test resourceVersion: "16978" uid: cfeb06b4-e61d-4d7b-89a3-5095f6ac6ebf kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:23Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq.18b92be839605d1e namespace: kserve-ci-e2e-test resourceVersion: "17007" uid: e58eb30f-170d-4273-bd57-4681acbb9e30 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq namespace: kserve-ci-e2e-test resourceVersion: "16978" uid: cfeb06b4-e61d-4d7b-89a3-5095f6ac6ebf kind: Event lastTimestamp: "2026-06-15T06:03:28Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-06-15T06:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:28Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq.18b92be946fcc1f0 namespace: kserve-ci-e2e-test resourceVersion: "17061" uid: 692988ef-4c17-4eaa-aae7-d21d77601b9a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq namespace: kserve-ci-e2e-test resourceVersion: "16978" uid: cfeb06b4-e61d-4d7b-89a3-5095f6ac6ebf kind: Event lastTimestamp: "2026-06-15T06:03:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:28Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq.18b92be94a2d0848 namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: d98f0e97-46ba-4a20-997e-cf4d8da4559b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq namespace: kserve-ci-e2e-test resourceVersion: "16978" uid: cfeb06b4-e61d-4d7b-89a3-5095f6ac6ebf kind: Event lastTimestamp: "2026-06-15T06:03:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:28Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq.18b92be94aac2112 namespace: kserve-ci-e2e-test resourceVersion: "17072" uid: ead2f5bf-659b-48ac-a301-c1b281f2a562 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq namespace: kserve-ci-e2e-test resourceVersion: "16978" uid: cfeb06b4-e61d-4d7b-89a3-5095f6ac6ebf kind: Event lastTimestamp: "2026-06-15T06:03:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:28Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq.18b92be94ab5f311 namespace: kserve-ci-e2e-test resourceVersion: "17073" uid: 18d9a58e-62ab-41f9-a4d1-b611bceded41 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq namespace: kserve-ci-e2e-test resourceVersion: "16978" uid: cfeb06b4-e61d-4d7b-89a3-5095f6ac6ebf kind: Event lastTimestamp: "2026-06-15T06:03:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:28Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq.18b92be94e13dd51 namespace: kserve-ci-e2e-test resourceVersion: "17076" uid: 9c9ca2cd-418c-4e2e-a500-4ec10d3363f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq namespace: kserve-ci-e2e-test resourceVersion: "16978" uid: cfeb06b4-e61d-4d7b-89a3-5095f6ac6ebf kind: Event lastTimestamp: "2026-06-15T06:03:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:28Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq.18b92be94ea5ca03 namespace: kserve-ci-e2e-test resourceVersion: "17078" uid: 4c6853b2-602e-4b67-9c09-bfce318efd84 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-06-15T06:03:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq namespace: kserve-ci-e2e-test resourceVersion: "16978" uid: cfeb06b4-e61d-4d7b-89a3-5095f6ac6ebf kind: Event lastTimestamp: "2026-06-15T06:05:15Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:03:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:15Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq.18b92be9834830f0 namespace: kserve-ci-e2e-test resourceVersion: "18018" uid: 128be965-34d1-49ab-aa84-9ff1e260e400 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq namespace: kserve-ci-e2e-test resourceVersion: "16978" uid: cfeb06b4-e61d-4d7b-89a3-5095f6ac6ebf kind: Event lastTimestamp: "2026-06-15T06:05:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:13Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq.18b92c01b997be38 namespace: kserve-ci-e2e-test resourceVersion: "17949" uid: fa9900de-5d35-4abf-8cb6-3028d30be169 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq namespace: kserve-ci-e2e-test resourceVersion: "16978" uid: cfeb06b4-e61d-4d7b-89a3-5095f6ac6ebf kind: Event lastTimestamp: "2026-06-15T06:05:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:13Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq.18b92c01b99834fa namespace: kserve-ci-e2e-test resourceVersion: "17952" uid: 39d0f8c9-9909-4d0a-82a9-0e3486b4ca07 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq namespace: kserve-ci-e2e-test resourceVersion: "16978" uid: cfeb06b4-e61d-4d7b-89a3-5095f6ac6ebf kind: Event lastTimestamp: "2026-06-15T06:05:15Z" message: 'Readiness probe failed: Get "https://10.132.0.23:8643/healthz": dial tcp 10.132.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:05:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:15Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq.18b92c02318bb190 namespace: kserve-ci-e2e-test resourceVersion: "18017" uid: 9ba5d1f1-f029-49a3-acd2-ec0cd7c2089b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 0705959e-fb0d-48b2-a8ef-68f4386bcbfc kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9-nf7sq' metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:23Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9.18b92be818ac782b namespace: kserve-ci-e2e-test resourceVersion: "16976" uid: 349f32b0-6b44-476a-b3ea-ca25bd193b56 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "16940" uid: 2f4a8689-eb99-48cb-a147-39730f7a64a6 kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-ea1cf-predictor-97f4566d9 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:23Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor.18b92be81140869e namespace: kserve-ci-e2e-test resourceVersion: "16949" uid: c803481f-4955-4320-bb7f-12c0c0bb1f10 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:03:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "16944" uid: 7e7fff59-0037-4d7e-ae5c-40e628994321 kind: Event lastTimestamp: "2026-06-15T06:04:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:03:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:04:08Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor.18b92beb8fa25592 namespace: kserve-ci-e2e-test resourceVersion: "17386" uid: 853073c3-05ba-47b0-8a08-7ea57077d5e1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:03:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "16944" uid: 7e7fff59-0037-4d7e-ae5c-40e628994321 kind: Event lastTimestamp: "2026-06-15T06:04:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:03:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:04:08Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor.18b92beb90029f36 namespace: kserve-ci-e2e-test resourceVersion: "17387" uid: ad367cdb-d3fb-4012-811d-8a942e69b1e4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:04:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "17163" uid: 7e7fff59-0037-4d7e-ae5c-40e628994321 kind: Event lastTimestamp: "2026-06-15T06:05:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:04:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:05:08Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor.18b92bf60bd176b8 namespace: kserve-ci-e2e-test resourceVersion: "17867" uid: e24034fd-6f44-4ed8-ba50-84c5e5dc160b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:04:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "17163" uid: 7e7fff59-0037-4d7e-ae5c-40e628994321 kind: Event lastTimestamp: "2026-06-15T06:05:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:04:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:05:08Z" name: isvc-xgboost-graph-raw-hpa-ea1cf-predictor.18b92bf60c344339 namespace: kserve-ci-e2e-test resourceVersion: "17868" uid: d1d14756-b3f5-4ef6-bd45-3e40c46500a8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-06-15T06:03:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-ea1cf namespace: kserve-ci-e2e-test resourceVersion: "16923" uid: fe90bc44-a917-49cb-bceb-1e99d987bb38 kind: Event lastTimestamp: "2026-06-15T06:04:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:04:35Z" name: isvc-xgboost-graph-raw-hpa-ea1cf.18b92be81119b8e6 namespace: kserve-ci-e2e-test resourceVersion: "17589" uid: 1a83dfef-c71f-418d-9f1d-ef8267cf7fc8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:04:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-ea1cf namespace: kserve-ci-e2e-test resourceVersion: "17587" uid: fe90bc44-a917-49cb-bceb-1e99d987bb38 kind: Event lastTimestamp: "2026-06-15T06:04:35Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-ea1cf] is Ready metadata: creationTimestamp: "2026-06-15T06:04:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:04:35Z" name: isvc-xgboost-graph-raw-hpa-ea1cf.18b92bf8e69c80ce namespace: kserve-ci-e2e-test resourceVersion: "17588" uid: ef6c1766-39a4-4d80-8e43-35b02536669b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:05:13.587007Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l namespace: kserve-ci-e2e-test resourceVersion: "17959" uid: 9e18deea-398e-4c1a-9f4b-900af722df84 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l to ip-10-0-135-167.ec2.internal metadata: creationTimestamp: "2026-06-15T06:05:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:05:13Z" name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l.18b92c01bd76de82 namespace: kserve-ci-e2e-test resourceVersion: "17966" uid: 65058f7b-f2ac-4181-ace8-2c8bc518025a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65bc5545-rprtl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:13Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l namespace: kserve-ci-e2e-test resourceVersion: "17960" uid: 9e18deea-398e-4c1a-9f4b-900af722df84 kind: Event lastTimestamp: "2026-06-15T06:05:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-raw-e564c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:13Z" name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l.18b92c01c925ec05 namespace: kserve-ci-e2e-test resourceVersion: "17994" uid: d8b122ab-e0ad-4354-8bd1-e13451efcd78 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:14Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l namespace: kserve-ci-e2e-test resourceVersion: "17973" uid: 9e18deea-398e-4c1a-9f4b-900af722df84 kind: Event lastTimestamp: "2026-06-15T06:05:14Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:05:14Z" name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l.18b92c01fad2f470 namespace: kserve-ci-e2e-test resourceVersion: "18008" uid: f1765a5a-a73a-44b6-a3f6-3cc19f43759a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l namespace: kserve-ci-e2e-test resourceVersion: "17960" uid: 9e18deea-398e-4c1a-9f4b-900af722df84 kind: Event lastTimestamp: "2026-06-15T06:05:14Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-06-15T06:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:14Z" name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l.18b92c01fc1554e1 namespace: kserve-ci-e2e-test resourceVersion: "18010" uid: 96064c03-d205-4423-9234-4fe0c2768a82 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l namespace: kserve-ci-e2e-test resourceVersion: "17960" uid: 9e18deea-398e-4c1a-9f4b-900af722df84 kind: Event lastTimestamp: "2026-06-15T06:05:15Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.311s (1.311s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-06-15T06:05:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:15Z" name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l.18b92c024a3bf2e6 namespace: kserve-ci-e2e-test resourceVersion: "18023" uid: 92f912ad-9b5b-49cd-b79b-54a52f55174f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l namespace: kserve-ci-e2e-test resourceVersion: "17960" uid: 9e18deea-398e-4c1a-9f4b-900af722df84 kind: Event lastTimestamp: "2026-06-15T06:05:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:05:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:15Z" name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l.18b92c024d2cd3c0 namespace: kserve-ci-e2e-test resourceVersion: "18024" uid: 6ac65310-15f8-43de-9863-9856f2a2a797 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l namespace: kserve-ci-e2e-test resourceVersion: "17960" uid: 9e18deea-398e-4c1a-9f4b-900af722df84 kind: Event lastTimestamp: "2026-06-15T06:05:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:05:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:16Z" name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l.18b92c024dac3d1d namespace: kserve-ci-e2e-test resourceVersion: "18025" uid: 471f1887-7386-469f-99a3-19ff2d18a72e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l namespace: kserve-ci-e2e-test resourceVersion: "17960" uid: 9e18deea-398e-4c1a-9f4b-900af722df84 kind: Event lastTimestamp: "2026-06-15T06:05:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:05:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:16Z" name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l.18b92c024db6e8c5 namespace: kserve-ci-e2e-test resourceVersion: "18026" uid: 5a4591fb-1004-4c36-b315-6744cd1d6b23 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l namespace: kserve-ci-e2e-test resourceVersion: "17960" uid: 9e18deea-398e-4c1a-9f4b-900af722df84 kind: Event lastTimestamp: "2026-06-15T06:05:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:05:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:16Z" name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l.18b92c02512743c2 namespace: kserve-ci-e2e-test resourceVersion: "18027" uid: 9a5ff396-aa8c-466f-846a-fc987415bd0f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l namespace: kserve-ci-e2e-test resourceVersion: "17960" uid: 9e18deea-398e-4c1a-9f4b-900af722df84 kind: Event lastTimestamp: "2026-06-15T06:05:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:05:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:16Z" name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l.18b92c02519d7fab namespace: kserve-ci-e2e-test resourceVersion: "18028" uid: 75d5bb4c-0f96-4c30-909c-936ae34e587f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l namespace: kserve-ci-e2e-test resourceVersion: "17960" uid: 9e18deea-398e-4c1a-9f4b-900af722df84 kind: Event lastTimestamp: "2026-06-15T06:07:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:07:08Z" name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l.18b92c1c9ae44ae4 namespace: kserve-ci-e2e-test resourceVersion: "19009" uid: bbbf3d46-28fb-4030-85a7-7712cb18c686 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l namespace: kserve-ci-e2e-test resourceVersion: "17960" uid: 9e18deea-398e-4c1a-9f4b-900af722df84 kind: Event lastTimestamp: "2026-06-15T06:07:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:07:08Z" name: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l.18b92c1c9ae57b5a namespace: kserve-ci-e2e-test resourceVersion: "19011" uid: c5c730e2-aafa-4e39-88a2-33dcb20b26d5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-e564c-predictor-5478dc4bc7 namespace: kserve-ci-e2e-test resourceVersion: "17951" uid: b9e161d3-bb67-4263-8287-689a3fea35b3 kind: Event lastTimestamp: "2026-06-15T06:05:13Z" message: 'Created pod: message-dumper-raw-e564c-predictor-5478dc4bc7-9nf6l' metadata: creationTimestamp: "2026-06-15T06:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:05:13Z" name: message-dumper-raw-e564c-predictor-5478dc4bc7.18b92c01bcd7bbc9 namespace: kserve-ci-e2e-test resourceVersion: "17962" uid: 14cf5cbb-6bc2-4814-b938-93e6a597c517 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-e564c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17947" uid: 5145b660-f594-41a5-ac65-865ac8b06181 kind: Event lastTimestamp: "2026-06-15T06:05:13Z" message: Scaled up replica set message-dumper-raw-e564c-predictor-5478dc4bc7 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:05:13Z" name: message-dumper-raw-e564c-predictor.18b92c01ba144033 namespace: kserve-ci-e2e-test resourceVersion: "17954" uid: bb57fd0d-1c82-475f-8fac-0db09d0117d7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-e564c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17950" uid: 5281caaa-69fc-477e-95ea-b7ffc869f636 kind: Event lastTimestamp: "2026-06-15T06:05:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:05:28Z" name: message-dumper-raw-e564c-predictor.18b92c05388bffb7 namespace: kserve-ci-e2e-test resourceVersion: "18139" uid: 988817e4-d6db-4134-9194-09c5872f96a4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-e564c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17950" uid: 5281caaa-69fc-477e-95ea-b7ffc869f636 kind: Event lastTimestamp: "2026-06-15T06:05:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:05:28Z" name: message-dumper-raw-e564c-predictor.18b92c0538f2738f namespace: kserve-ci-e2e-test resourceVersion: "18141" uid: d28b6a30-f801-457c-97e7-88e033138569 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-e564c-predictor namespace: kserve-ci-e2e-test resourceVersion: "18140" uid: 5281caaa-69fc-477e-95ea-b7ffc869f636 kind: Event lastTimestamp: "2026-06-15T06:05:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:05:43Z" name: message-dumper-raw-e564c-predictor.18b92c08b7faa48e namespace: kserve-ci-e2e-test resourceVersion: "18314" uid: 29a5f719-6e3e-4a7a-8fac-0703f9a4d0a3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-e564c-predictor namespace: kserve-ci-e2e-test resourceVersion: "18140" uid: 5281caaa-69fc-477e-95ea-b7ffc869f636 kind: Event lastTimestamp: "2026-06-15T06:05:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:05:43Z" name: message-dumper-raw-e564c-predictor.18b92c08b8605562 namespace: kserve-ci-e2e-test resourceVersion: "18316" uid: 823060b7-4526-4356-9ea3-87e7c03b84c6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-06-15T06:05:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-e564c namespace: kserve-ci-e2e-test resourceVersion: "17924" uid: 745a40cb-37d7-470e-a54e-2356189dc053 kind: Event lastTimestamp: "2026-06-15T06:05:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:05:25Z" name: message-dumper-raw-e564c.18b92c01b9f1834c namespace: kserve-ci-e2e-test resourceVersion: "18113" uid: bc53293a-4709-4883-a98b-02440c06b467 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:05:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-e564c namespace: kserve-ci-e2e-test resourceVersion: "17924" uid: 745a40cb-37d7-470e-a54e-2356189dc053 kind: Event lastTimestamp: "2026-06-15T06:05:13Z" message: 'Failed to update status for InferenceService "message-dumper-raw-e564c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-e564c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:05:13Z" name: message-dumper-raw-e564c.18b92c01ba43297f namespace: kserve-ci-e2e-test resourceVersion: "17990" uid: 960d9b57-0a22-4a2b-93d1-2508d009f91a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-e564c namespace: kserve-ci-e2e-test resourceVersion: "17936" uid: 745a40cb-37d7-470e-a54e-2356189dc053 kind: Event lastTimestamp: "2026-06-15T06:05:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-e564c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:05:13Z" name: message-dumper-raw-e564c.18b92c01c162ff67 namespace: kserve-ci-e2e-test resourceVersion: "17992" uid: 6a467256-af48-47e8-8e6b-e3358ef91ad2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-e564c namespace: kserve-ci-e2e-test resourceVersion: "18110" uid: 745a40cb-37d7-470e-a54e-2356189dc053 kind: Event lastTimestamp: "2026-06-15T06:05:25Z" message: InferenceService [message-dumper-raw-e564c] is Ready metadata: creationTimestamp: "2026-06-15T06:05:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:05:25Z" name: message-dumper-raw-e564c.18b92c046759a5fd namespace: kserve-ci-e2e-test resourceVersion: "18112" uid: e7bfb3a1-6ff7-47a8-9875-6cff28fe3199 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:03:13.014685Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-2af6c-759f8bd49d-2fsnv namespace: kserve-ci-e2e-test resourceVersion: "16765" uid: ab25b01f-6453-4003-8d52-c55f6b464fc7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-2af6c-759f8bd49d-2fsnv to ip-10-0-136-138.ec2.internal metadata: creationTimestamp: "2026-06-15T06:03:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:03:13Z" name: model-chainer-raw-2af6c-759f8bd49d-2fsnv.18b92be5aacb34b3 namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: 73882bd7-3bb3-4ace-83aa-27007a8ab007 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65bc5545-rprtl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:13Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-2af6c-759f8bd49d-2fsnv namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: ab25b01f-6453-4003-8d52-c55f6b464fc7 kind: Event lastTimestamp: "2026-06-15T06:03:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-2af6c-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:03:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:13Z" name: model-chainer-raw-2af6c-759f8bd49d-2fsnv.18b92be5b6028f07 namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 3c16e33a-a850-46c1-8b55-24a0d47828e5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:14Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-2af6c-759f8bd49d-2fsnv namespace: kserve-ci-e2e-test resourceVersion: "16774" uid: ab25b01f-6453-4003-8d52-c55f6b464fc7 kind: Event lastTimestamp: "2026-06-15T06:03:14Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:03:14Z" name: model-chainer-raw-2af6c-759f8bd49d-2fsnv.18b92be5e96e2360 namespace: kserve-ci-e2e-test resourceVersion: "16790" uid: e672b7a5-764d-441b-bc71-87f4513e56b4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-2af6c} kind: Pod name: model-chainer-raw-2af6c-759f8bd49d-2fsnv namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: ab25b01f-6453-4003-8d52-c55f6b464fc7 kind: Event lastTimestamp: "2026-06-15T06:03:14Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:c92ab94c818c88d4324be5bb5a3d9bda3e4173010dc47851ff3051c5a06f1614" metadata: creationTimestamp: "2026-06-15T06:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:14Z" name: model-chainer-raw-2af6c-759f8bd49d-2fsnv.18b92be5eab0df46 namespace: kserve-ci-e2e-test resourceVersion: "16793" uid: cf7ebad6-48c4-430d-93ea-a7f974581f61 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-2af6c} kind: Pod name: model-chainer-raw-2af6c-759f8bd49d-2fsnv namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: ab25b01f-6453-4003-8d52-c55f6b464fc7 kind: Event lastTimestamp: "2026-06-15T06:03:16Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:c92ab94c818c88d4324be5bb5a3d9bda3e4173010dc47851ff3051c5a06f1614" in 2.566s (2.566s including waiting). Image size: 218065501 bytes.' metadata: creationTimestamp: "2026-06-15T06:03:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:16Z" name: model-chainer-raw-2af6c-759f8bd49d-2fsnv.18b92be683ad54c1 namespace: kserve-ci-e2e-test resourceVersion: "16812" uid: b1768e1f-804f-4f77-888a-3118067dfe35 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-2af6c} kind: Pod name: model-chainer-raw-2af6c-759f8bd49d-2fsnv namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: ab25b01f-6453-4003-8d52-c55f6b464fc7 kind: Event lastTimestamp: "2026-06-15T06:03:16Z" message: 'Created container: model-chainer-raw-2af6c' metadata: creationTimestamp: "2026-06-15T06:03:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:16Z" name: model-chainer-raw-2af6c-759f8bd49d-2fsnv.18b92be686dd0672 namespace: kserve-ci-e2e-test resourceVersion: "16813" uid: 0441824b-9529-4ad2-8240-e3f0c57983f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-2af6c} kind: Pod name: model-chainer-raw-2af6c-759f8bd49d-2fsnv namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: ab25b01f-6453-4003-8d52-c55f6b464fc7 kind: Event lastTimestamp: "2026-06-15T06:03:16Z" message: Started container model-chainer-raw-2af6c metadata: creationTimestamp: "2026-06-15T06:03:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:16Z" name: model-chainer-raw-2af6c-759f8bd49d-2fsnv.18b92be68757333e namespace: kserve-ci-e2e-test resourceVersion: "16814" uid: f6458459-5512-4651-84f3-b41607d4e57c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-2af6c} kind: Pod name: model-chainer-raw-2af6c-759f8bd49d-2fsnv namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: ab25b01f-6453-4003-8d52-c55f6b464fc7 kind: Event lastTimestamp: "2026-06-15T06:03:23Z" message: Stopping container model-chainer-raw-2af6c metadata: creationTimestamp: "2026-06-15T06:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:23Z" name: model-chainer-raw-2af6c-759f8bd49d-2fsnv.18b92be805b63f84 namespace: kserve-ci-e2e-test resourceVersion: "16887" uid: 87d9103c-7ff4-4558-9c96-4d90d4e6caef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:03:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-2af6c} kind: Pod name: model-chainer-raw-2af6c-759f8bd49d-2fsnv namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: ab25b01f-6453-4003-8d52-c55f6b464fc7 kind: Event lastTimestamp: "2026-06-15T06:03:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-06-15T06:03:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:52Z" name: model-chainer-raw-2af6c-759f8bd49d-2fsnv.18b92be91c206cae namespace: kserve-ci-e2e-test resourceVersion: "17270" uid: d5a85bff-7ed1-46fc-836c-2f1bfe5e64bd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-2af6c-759f8bd49d namespace: kserve-ci-e2e-test resourceVersion: "16762" uid: c8b2b3eb-6fe7-4a9f-a1da-a37d98cbca9f kind: Event lastTimestamp: "2026-06-15T06:03:13Z" message: 'Created pod: model-chainer-raw-2af6c-759f8bd49d-2fsnv' metadata: creationTimestamp: "2026-06-15T06:03:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:13Z" name: model-chainer-raw-2af6c-759f8bd49d.18b92be5aa619895 namespace: kserve-ci-e2e-test resourceVersion: "16767" uid: 8c3665b8-6cf9-4785-acc2-6e6341b7e929 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-2af6c namespace: kserve-ci-e2e-test resourceVersion: "16760" uid: 79c8f219-4835-47fc-9f8a-0fc792cb64cb kind: Event lastTimestamp: "2026-06-15T06:03:12Z" message: Scaled up replica set model-chainer-raw-2af6c-759f8bd49d from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:03:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:12Z" name: model-chainer-raw-2af6c.18b92be5a9729096 namespace: kserve-ci-e2e-test resourceVersion: "16763" uid: 4b6e5a55-14c2-480b-8c39-624cb427a7f6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-2af6c namespace: kserve-ci-e2e-test resourceVersion: "16777" uid: f1c7fe2c-bce1-4b90-84cf-b1716436ed2c kind: Event lastTimestamp: "2026-06-15T06:03:13Z" message: InferenceGraph [model-chainer-raw-2af6c] is Ready metadata: creationTimestamp: "2026-06-15T06:03:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:03:13Z" name: model-chainer-raw-2af6c.18b92be5aea8eaa1 namespace: kserve-ci-e2e-test resourceVersion: "16779" uid: ef0353fd-93fa-4d28-bea3-9fe9429dbe23 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:05:03.354657Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-ea1cf-6b485bd866-m4xtt namespace: kserve-ci-e2e-test resourceVersion: "17804" uid: 24e96cf8-00f8-4fcb-b3ce-e5d43434bc51 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-ea1cf-6b485bd866-m4xtt to ip-10-0-136-138.ec2.internal metadata: creationTimestamp: "2026-06-15T06:05:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:05:03Z" name: model-chainer-raw-hpa-ea1cf-6b485bd866-m4xtt.18b92bff5b91a1d9 namespace: kserve-ci-e2e-test resourceVersion: "17810" uid: 1da83b9a-06cf-4a3a-b8da-a8da139c3cd3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65bc5545-rprtl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:03Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-ea1cf-6b485bd866-m4xtt namespace: kserve-ci-e2e-test resourceVersion: "17807" uid: 24e96cf8-00f8-4fcb-b3ce-e5d43434bc51 kind: Event lastTimestamp: "2026-06-15T06:05:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-ea1cf-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:05:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:03Z" name: model-chainer-raw-hpa-ea1cf-6b485bd866-m4xtt.18b92bff665ecebe namespace: kserve-ci-e2e-test resourceVersion: "17821" uid: 80809bec-7880-4399-9001-b104bfa2c6ef reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:04Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-ea1cf-6b485bd866-m4xtt namespace: kserve-ci-e2e-test resourceVersion: "17813" uid: 24e96cf8-00f8-4fcb-b3ce-e5d43434bc51 kind: Event lastTimestamp: "2026-06-15T06:05:04Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:05:04Z" name: model-chainer-raw-hpa-ea1cf-6b485bd866-m4xtt.18b92bff98f841ab namespace: kserve-ci-e2e-test resourceVersion: "17831" uid: c0892dd7-0f25-4dcc-b5a1-3a02575b6501 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-ea1cf} kind: Pod name: model-chainer-raw-hpa-ea1cf-6b485bd866-m4xtt namespace: kserve-ci-e2e-test resourceVersion: "17807" uid: 24e96cf8-00f8-4fcb-b3ce-e5d43434bc51 kind: Event lastTimestamp: "2026-06-15T06:05:04Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c92ab94c818c88d4324be5bb5a3d9bda3e4173010dc47851ff3051c5a06f1614" already present on machine metadata: creationTimestamp: "2026-06-15T06:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:04Z" name: model-chainer-raw-hpa-ea1cf-6b485bd866-m4xtt.18b92bff9a257e5b namespace: kserve-ci-e2e-test resourceVersion: "17833" uid: fef2a7e9-f783-433e-ab32-d5f7ea2edd7c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-ea1cf} kind: Pod name: model-chainer-raw-hpa-ea1cf-6b485bd866-m4xtt namespace: kserve-ci-e2e-test resourceVersion: "17807" uid: 24e96cf8-00f8-4fcb-b3ce-e5d43434bc51 kind: Event lastTimestamp: "2026-06-15T06:05:04Z" message: 'Created container: model-chainer-raw-hpa-ea1cf' metadata: creationTimestamp: "2026-06-15T06:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:04Z" name: model-chainer-raw-hpa-ea1cf-6b485bd866-m4xtt.18b92bff9d5070a0 namespace: kserve-ci-e2e-test resourceVersion: "17834" uid: 7288a634-6968-4d2f-9fa6-5d5b9a53a719 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-ea1cf} kind: Pod name: model-chainer-raw-hpa-ea1cf-6b485bd866-m4xtt namespace: kserve-ci-e2e-test resourceVersion: "17807" uid: 24e96cf8-00f8-4fcb-b3ce-e5d43434bc51 kind: Event lastTimestamp: "2026-06-15T06:05:04Z" message: Started container model-chainer-raw-hpa-ea1cf metadata: creationTimestamp: "2026-06-15T06:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:04Z" name: model-chainer-raw-hpa-ea1cf-6b485bd866-m4xtt.18b92bff9db937dd namespace: kserve-ci-e2e-test resourceVersion: "17835" uid: 641a419e-58e7-4e6c-96da-a5cdbaa98b63 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-ea1cf} kind: Pod name: model-chainer-raw-hpa-ea1cf-6b485bd866-m4xtt namespace: kserve-ci-e2e-test resourceVersion: "17807" uid: 24e96cf8-00f8-4fcb-b3ce-e5d43434bc51 kind: Event lastTimestamp: "2026-06-15T06:05:13Z" message: Stopping container model-chainer-raw-hpa-ea1cf metadata: creationTimestamp: "2026-06-15T06:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:13Z" name: model-chainer-raw-hpa-ea1cf-6b485bd866-m4xtt.18b92c01b30c6167 namespace: kserve-ci-e2e-test resourceVersion: "17919" uid: 7d429dd3-6651-4759-b29a-2afc5664e2b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:05:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-ea1cf} kind: Pod name: model-chainer-raw-hpa-ea1cf-6b485bd866-m4xtt namespace: kserve-ci-e2e-test resourceVersion: "17807" uid: 24e96cf8-00f8-4fcb-b3ce-e5d43434bc51 kind: Event lastTimestamp: "2026-06-15T06:05:41Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-06-15T06:05:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:05:41Z" name: model-chainer-raw-hpa-ea1cf-6b485bd866-m4xtt.18b92c02601d5ad1 namespace: kserve-ci-e2e-test resourceVersion: "18299" uid: 45615197-852c-4e28-b308-c94542f19b29 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-ea1cf-6b485bd866 namespace: kserve-ci-e2e-test resourceVersion: "17801" uid: 67e23f35-c8a3-4905-a776-8c1961344454 kind: Event lastTimestamp: "2026-06-15T06:05:03Z" message: 'Created pod: model-chainer-raw-hpa-ea1cf-6b485bd866-m4xtt' metadata: creationTimestamp: "2026-06-15T06:05:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:05:03Z" name: model-chainer-raw-hpa-ea1cf-6b485bd866.18b92bff5b0c2d26 namespace: kserve-ci-e2e-test resourceVersion: "17806" uid: 1b179e9b-8664-4f50-b234-9b340b3b0640 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-ea1cf namespace: kserve-ci-e2e-test resourceVersion: "17799" uid: 38c88874-6615-4d64-96de-c0c1f213c1da kind: Event lastTimestamp: "2026-06-15T06:05:03Z" message: Scaled up replica set model-chainer-raw-hpa-ea1cf-6b485bd866 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:05:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:05:03Z" name: model-chainer-raw-hpa-ea1cf.18b92bff5a162200 namespace: kserve-ci-e2e-test resourceVersion: "17802" uid: c061192a-d5c5-4526-a992-9b8206205d59 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:05:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-ea1cf namespace: kserve-ci-e2e-test resourceVersion: "17815" uid: 0500bc6e-b458-47b3-945e-9afb1f32bc38 kind: Event lastTimestamp: "2026-06-15T06:05:03Z" message: InferenceGraph [model-chainer-raw-hpa-ea1cf] is Ready metadata: creationTimestamp: "2026-06-15T06:05:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:05:03Z" name: model-chainer-raw-hpa-ea1cf.18b92bff5f73a0ce namespace: kserve-ci-e2e-test resourceVersion: "17817" uid: 7d3f7665-c462-48b7-855d-cb34286bec0e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:11:52.708028Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7 namespace: kserve-ci-e2e-test resourceVersion: "21350" uid: 79511fbe-a197-4b18-add3-d6d5d81a2f32 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7 to ip-10-0-135-167.ec2.internal metadata: creationTimestamp: "2026-06-15T06:11:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:11:52Z" name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7.18b92c5eaaee57bd namespace: kserve-ci-e2e-test resourceVersion: "21354" uid: e2e5c0ba-b539-47d1-8988-0da4bc02dc18 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65bc5545-rprtl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:53Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7 namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: 79511fbe-a197-4b18-add3-d6d5d81a2f32 kind: Event lastTimestamp: "2026-06-15T06:11:53Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:11:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:11:53Z" name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7.18b92c5ec4eb0580 namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: 3350cfb7-7e73-47ef-aba1-bec78dfefedf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7 namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 79511fbe-a197-4b18-add3-d6d5d81a2f32 kind: Event lastTimestamp: "2026-06-15T06:11:53Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ba8edcbfb3f9312d158be16483785d7654e60c7090f262c42214fd2b29effada" already present on machine metadata: creationTimestamp: "2026-06-15T06:11:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:53Z" name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7.18b92c5ec62f7c4b namespace: kserve-ci-e2e-test resourceVersion: "21372" uid: be2f8a57-42e1-4e8e-9eea-9f3f4dfd32ac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7 namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 79511fbe-a197-4b18-add3-d6d5d81a2f32 kind: Event lastTimestamp: "2026-06-15T06:11:53Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-15T06:11:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:53Z" name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7.18b92c5ec9c01f27 namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: 6db76995-d621-478c-a4bd-0067bcdd7721 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7 namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 79511fbe-a197-4b18-add3-d6d5d81a2f32 kind: Event lastTimestamp: "2026-06-15T06:11:53Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-15T06:11:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:53Z" name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7.18b92c5eca32a3a9 namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: 03cf1759-50fe-4e20-b531-a6dde1f8faea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7 namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 79511fbe-a197-4b18-add3-d6d5d81a2f32 kind: Event lastTimestamp: "2026-06-15T06:11:57Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:11:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:57Z" name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7.18b92c5fc702f8fa namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: af3e2c52-12c9-4da1-ae7b-0f0f81639270 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7 namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 79511fbe-a197-4b18-add3-d6d5d81a2f32 kind: Event lastTimestamp: "2026-06-15T06:11:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:11:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:57Z" name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7.18b92c5fca969bf3 namespace: kserve-ci-e2e-test resourceVersion: "21430" uid: 23f2f5ff-d7bf-4ce1-9329-558577e1fc81 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7 namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 79511fbe-a197-4b18-add3-d6d5d81a2f32 kind: Event lastTimestamp: "2026-06-15T06:11:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:11:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:57Z" name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7.18b92c5fcb1559b7 namespace: kserve-ci-e2e-test resourceVersion: "21432" uid: 61af1111-8721-4a2a-8020-c0dfb653652a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7 namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 79511fbe-a197-4b18-add3-d6d5d81a2f32 kind: Event lastTimestamp: "2026-06-15T06:11:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:11:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:57Z" name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7.18b92c5fcb21cab4 namespace: kserve-ci-e2e-test resourceVersion: "21433" uid: 675f7d97-e526-4dd6-a81e-f1f27f058340 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7 namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 79511fbe-a197-4b18-add3-d6d5d81a2f32 kind: Event lastTimestamp: "2026-06-15T06:11:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:11:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:57Z" name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7.18b92c5fce315b2d namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: 54bde3e3-e31c-4649-9efa-8bca49691936 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7 namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 79511fbe-a197-4b18-add3-d6d5d81a2f32 kind: Event lastTimestamp: "2026-06-15T06:11:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:11:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:57Z" name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7.18b92c5fceaaa4b4 namespace: kserve-ci-e2e-test resourceVersion: "21437" uid: a4f991ff-dbc3-47a8-b535-c47a0413b85b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-06-15T06:11:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7 namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 79511fbe-a197-4b18-add3-d6d5d81a2f32 kind: Event lastTimestamp: "2026-06-15T06:13:26Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:26Z" name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7.18b92c603ecbc050 namespace: kserve-ci-e2e-test resourceVersion: "22136" uid: 3cd6673e-3373-45f8-ba20-ba4f59f5a5c4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7 namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 79511fbe-a197-4b18-add3-d6d5d81a2f32 kind: Event lastTimestamp: "2026-06-15T06:13:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:22Z" name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7.18b92c73ac8d9c43 namespace: kserve-ci-e2e-test resourceVersion: "22063" uid: e7434a04-612c-4050-a7f3-8938d6e2e144 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7 namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 79511fbe-a197-4b18-add3-d6d5d81a2f32 kind: Event lastTimestamp: "2026-06-15T06:13:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:22Z" name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7.18b92c73ac8e43a2 namespace: kserve-ci-e2e-test resourceVersion: "22064" uid: 50e5eebf-0c30-410e-b7b8-95ecdf3ece71 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7 namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 79511fbe-a197-4b18-add3-d6d5d81a2f32 kind: Event lastTimestamp: "2026-06-15T06:13:25Z" message: 'Readiness probe failed: Get "https://10.132.0.30:8643/healthz": dial tcp 10.132.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:13:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:25Z" name: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7.18b92c7444ec926e namespace: kserve-ci-e2e-test resourceVersion: "22130" uid: 2c33a71a-643f-45d2-85a7-81b11e88c7f3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-6fcbc-predictor-76cb49bc97 namespace: kserve-ci-e2e-test resourceVersion: "21340" uid: 3d8036ea-d3a9-460a-9672-4924481d3701 kind: Event lastTimestamp: "2026-06-15T06:11:52Z" message: 'Created pod: raw-sklearn-6fcbc-predictor-76cb49bc97-2x4x7' metadata: creationTimestamp: "2026-06-15T06:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:52Z" name: raw-sklearn-6fcbc-predictor-76cb49bc97.18b92c5eaa846720 namespace: kserve-ci-e2e-test resourceVersion: "21351" uid: 2a95aacf-0ef1-46a3-8495-0395e004071f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-6fcbc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21338" uid: ca40ef3a-1e60-442b-8a31-003b546f4800 kind: Event lastTimestamp: "2026-06-15T06:11:52Z" message: Scaled up replica set raw-sklearn-6fcbc-predictor-76cb49bc97 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:52Z" name: raw-sklearn-6fcbc-predictor.18b92c5ea7ea9aac namespace: kserve-ci-e2e-test resourceVersion: "21342" uid: 3c15479b-58d4-4b29-b05f-92c636fa2b04 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:12:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-6fcbc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21339" uid: 123dc493-9338-42a6-8633-24c3fec90410 kind: Event lastTimestamp: "2026-06-15T06:12:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:12:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:52Z" name: raw-sklearn-6fcbc-predictor.18b92c6226b03ed6 namespace: kserve-ci-e2e-test resourceVersion: "21821" uid: 010dccbe-2375-40c0-aae1-283c496919bf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:12:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-6fcbc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21339" uid: 123dc493-9338-42a6-8633-24c3fec90410 kind: Event lastTimestamp: "2026-06-15T06:12:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:12:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:52Z" name: raw-sklearn-6fcbc-predictor.18b92c6227152002 namespace: kserve-ci-e2e-test resourceVersion: "21823" uid: 75fb6463-b369-479e-b987-c3e8875663ec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:13:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-6fcbc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21511" uid: 123dc493-9338-42a6-8633-24c3fec90410 kind: Event lastTimestamp: "2026-06-15T06:13:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:13:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:22Z" name: raw-sklearn-6fcbc-predictor.18b92c70227f3165 namespace: kserve-ci-e2e-test resourceVersion: "22044" uid: 2432d37c-4ef2-4fa9-ab05-3aae92f7473e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:13:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-6fcbc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21511" uid: 123dc493-9338-42a6-8633-24c3fec90410 kind: Event lastTimestamp: "2026-06-15T06:13:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:13:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:22Z" name: raw-sklearn-6fcbc-predictor.18b92c7022e301f5 namespace: kserve-ci-e2e-test resourceVersion: "22045" uid: 32e5bd2f-a791-43ce-a008-a9cf07c83504 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-06-15T06:11:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-6fcbc namespace: kserve-ci-e2e-test resourceVersion: "21328" uid: 7b2053ff-ed8e-453b-a9d0-7f9bedccca62 kind: Event lastTimestamp: "2026-06-15T06:13:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:13:16Z" name: raw-sklearn-6fcbc.18b92c5ea7c96e57 namespace: kserve-ci-e2e-test resourceVersion: "21996" uid: 8aaf642a-9b03-4713-9045-b6ba3405023c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:11:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-6fcbc namespace: kserve-ci-e2e-test resourceVersion: "21328" uid: 7b2053ff-ed8e-453b-a9d0-7f9bedccca62 kind: Event lastTimestamp: "2026-06-15T06:11:53Z" message: 'Failed to update status for InferenceService "raw-sklearn-6fcbc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-6fcbc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:53Z" name: raw-sklearn-6fcbc.18b92c5ea8304046 namespace: kserve-ci-e2e-test resourceVersion: "21390" uid: efc461cf-e6a4-45e6-b85d-96388fe97950 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:11:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-6fcbc namespace: kserve-ci-e2e-test resourceVersion: "21348" uid: 7b2053ff-ed8e-453b-a9d0-7f9bedccca62 kind: Event lastTimestamp: "2026-06-15T06:11:53Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-6fcbc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:11:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:53Z" name: raw-sklearn-6fcbc.18b92c5eaf45cf54 namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: 8942b318-c685-43a5-a9a7-6d8882f4471c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-6fcbc namespace: kserve-ci-e2e-test resourceVersion: "21993" uid: 7b2053ff-ed8e-453b-a9d0-7f9bedccca62 kind: Event lastTimestamp: "2026-06-15T06:13:16Z" message: InferenceService [raw-sklearn-6fcbc] is Ready metadata: creationTimestamp: "2026-06-15T06:13:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:13:16Z" name: raw-sklearn-6fcbc.18b92c721dec5fdd namespace: kserve-ci-e2e-test resourceVersion: "21994" uid: a1d0f576-862e-4e2f-a12b-1e79362fd76e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:13:23.071083Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: 548f6155-f398-4ace-a589-722d96b7c487 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd to ip-10-0-135-167.ec2.internal metadata: creationTimestamp: "2026-06-15T06:13:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:13:23Z" name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd.18b92c73b4fd2b2d namespace: kserve-ci-e2e-test resourceVersion: "22092" uid: 159ae409-89f7-434a-80cb-e908c7108f89 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-65bc5545-rprtl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:23Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd namespace: kserve-ci-e2e-test resourceVersion: "22097" uid: 548f6155-f398-4ace-a589-722d96b7c487 kind: Event lastTimestamp: "2026-06-15T06:13:23Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:13:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:13:23Z" name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd.18b92c73ceceafd0 namespace: kserve-ci-e2e-test resourceVersion: "22105" uid: ca6df295-d7e3-4eea-9c32-f22b52270e35 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: 548f6155-f398-4ace-a589-722d96b7c487 kind: Event lastTimestamp: "2026-06-15T06:13:23Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ba8edcbfb3f9312d158be16483785d7654e60c7090f262c42214fd2b29effada" already present on machine metadata: creationTimestamp: "2026-06-15T06:13:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:23Z" name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd.18b92c73d02f4602 namespace: kserve-ci-e2e-test resourceVersion: "22109" uid: 07253676-04d6-43bd-b4a6-f34f32a2ad46 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: 548f6155-f398-4ace-a589-722d96b7c487 kind: Event lastTimestamp: "2026-06-15T06:13:23Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-15T06:13:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:23Z" name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd.18b92c73d3150d3d namespace: kserve-ci-e2e-test resourceVersion: "22110" uid: a300483c-daa2-4cb4-ac34-42e5c23ec534 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: 548f6155-f398-4ace-a589-722d96b7c487 kind: Event lastTimestamp: "2026-06-15T06:13:23Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-15T06:13:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:23Z" name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd.18b92c73d38c26dd namespace: kserve-ci-e2e-test resourceVersion: "22111" uid: efe3d918-17bf-4723-804d-bd207bc618a1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: 548f6155-f398-4ace-a589-722d96b7c487 kind: Event lastTimestamp: "2026-06-15T06:13:27Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:13:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:27Z" name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd.18b92c74cfa1b66a namespace: kserve-ci-e2e-test resourceVersion: "22145" uid: 1f80a62b-9d2e-4f67-8ea4-7eac9145a881 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: 548f6155-f398-4ace-a589-722d96b7c487 kind: Event lastTimestamp: "2026-06-15T06:13:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:13:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:27Z" name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd.18b92c74d64ba020 namespace: kserve-ci-e2e-test resourceVersion: "22153" uid: ec3d3106-8c38-4c2a-a433-730023f71159 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: 548f6155-f398-4ace-a589-722d96b7c487 kind: Event lastTimestamp: "2026-06-15T06:13:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:13:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:27Z" name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd.18b92c74d6df38cf namespace: kserve-ci-e2e-test resourceVersion: "22155" uid: 13c6a794-b79d-4f6b-a25b-f8e52bb5e23e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: 548f6155-f398-4ace-a589-722d96b7c487 kind: Event lastTimestamp: "2026-06-15T06:13:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:13:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:27Z" name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd.18b92c74d6e9ea16 namespace: kserve-ci-e2e-test resourceVersion: "22156" uid: 1131b506-f11c-443f-9f75-2980043fcb14 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: 548f6155-f398-4ace-a589-722d96b7c487 kind: Event lastTimestamp: "2026-06-15T06:13:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:13:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:27Z" name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd.18b92c74d9ef1792 namespace: kserve-ci-e2e-test resourceVersion: "22157" uid: 673f9e5c-9cb9-4321-9365-79798344a19f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: 548f6155-f398-4ace-a589-722d96b7c487 kind: Event lastTimestamp: "2026-06-15T06:13:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:13:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:27Z" name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd.18b92c74da6a1b1c namespace: kserve-ci-e2e-test resourceVersion: "22158" uid: f73f1361-25f5-4073-ad1f-c6c50fb51562 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-06-15T06:13:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: 548f6155-f398-4ace-a589-722d96b7c487 kind: Event lastTimestamp: "2026-06-15T06:14:54Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:13:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:14:54Z" name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd.18b92c750bca5438 namespace: kserve-ci-e2e-test resourceVersion: "22835" uid: 693f74ef-0593-41a1-b244-4ce497ac444f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:14:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: 548f6155-f398-4ace-a589-722d96b7c487 kind: Event lastTimestamp: "2026-06-15T06:14:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:14:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:14:53Z" name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd.18b92c88adcfec1a namespace: kserve-ci-e2e-test resourceVersion: "22791" uid: 025c8657-0913-44c1-9e1a-38023cf61b04 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:14:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: 548f6155-f398-4ace-a589-722d96b7c487 kind: Event lastTimestamp: "2026-06-15T06:14:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:14:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:14:53Z" name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd.18b92c88add06ed7 namespace: kserve-ci-e2e-test resourceVersion: "22792" uid: b601c040-dfdd-4989-8cd4-a092fa0ada54 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:14:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: 548f6155-f398-4ace-a589-722d96b7c487 kind: Event lastTimestamp: "2026-06-15T06:14:54Z" message: 'Readiness probe failed: Get "https://10.132.0.31:8643/healthz": dial tcp 10.132.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:14:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:14:54Z" name: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd.18b92c89120466ed namespace: kserve-ci-e2e-test resourceVersion: "22834" uid: e585a65c-6813-467c-85f4-5c64d712724d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-167.ec2.internal source: component: kubelet host: ip-10-0-135-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-34378-predictor-7679fff757 namespace: kserve-ci-e2e-test resourceVersion: "22076" uid: ab36e979-a9d9-4bd4-9f88-8d6bae7522e5 kind: Event lastTimestamp: "2026-06-15T06:13:23Z" message: 'Created pod: raw-sklearn-runtime-34378-predictor-7679fff757-jfhxd' metadata: creationTimestamp: "2026-06-15T06:13:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:23Z" name: raw-sklearn-runtime-34378-predictor-7679fff757.18b92c73b47421b7 namespace: kserve-ci-e2e-test resourceVersion: "22089" uid: e33f2968-71fb-4042-8dbc-325c1858fd55 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-34378-predictor namespace: kserve-ci-e2e-test resourceVersion: "22073" uid: ddbb0e12-7e67-45b3-a726-6c3db2f08de3 kind: Event lastTimestamp: "2026-06-15T06:13:22Z" message: Scaled up replica set raw-sklearn-runtime-34378-predictor-7679fff757 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:22Z" name: raw-sklearn-runtime-34378-predictor.18b92c73af4333ec namespace: kserve-ci-e2e-test resourceVersion: "22078" uid: e9feb00d-42af-4edf-b958-b6b4931102e2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:13:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-34378-predictor namespace: kserve-ci-e2e-test resourceVersion: "22075" uid: ebc67c75-c490-4cb5-8642-d6bb72223706 kind: Event lastTimestamp: "2026-06-15T06:14:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:13:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:14:08Z" name: raw-sklearn-runtime-34378-predictor.18b92c772e2202a2 namespace: kserve-ci-e2e-test resourceVersion: "22450" uid: 4ed2c707-d179-493a-b12b-9484988ab131 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:13:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-34378-predictor namespace: kserve-ci-e2e-test resourceVersion: "22075" uid: ebc67c75-c490-4cb5-8642-d6bb72223706 kind: Event lastTimestamp: "2026-06-15T06:14:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:13:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:14:08Z" name: raw-sklearn-runtime-34378-predictor.18b92c772e86e058 namespace: kserve-ci-e2e-test resourceVersion: "22451" uid: 70ae1a65-fc71-4cb9-bce1-d1f262e84429 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:13:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-34378-predictor namespace: kserve-ci-e2e-test resourceVersion: "22236" uid: ebc67c75-c490-4cb5-8642-d6bb72223706 kind: Event lastTimestamp: "2026-06-15T06:14:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:13:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:14:53Z" name: raw-sklearn-runtime-34378-predictor.18b92c7aad5e7d37 namespace: kserve-ci-e2e-test resourceVersion: "22774" uid: 5ecfc303-25ac-40b7-80a4-72c21e387660 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:13:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-34378-predictor namespace: kserve-ci-e2e-test resourceVersion: "22236" uid: ebc67c75-c490-4cb5-8642-d6bb72223706 kind: Event lastTimestamp: "2026-06-15T06:14:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:13:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:14:53Z" name: raw-sklearn-runtime-34378-predictor.18b92c7aada5a803 namespace: kserve-ci-e2e-test resourceVersion: "22775" uid: 07fd0b83-07cc-491a-9c3c-34d0a068b0ff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-06-15T06:13:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-34378 namespace: kserve-ci-e2e-test resourceVersion: "22055" uid: 2bcdb93a-a728-456a-8849-2ab1845d2321 kind: Event lastTimestamp: "2026-06-15T06:14:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:14:44Z" name: raw-sklearn-runtime-34378.18b92c73af219806 namespace: kserve-ci-e2e-test resourceVersion: "22718" uid: 79d688da-a122-4f43-88a9-5881bbda961a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-34378 namespace: kserve-ci-e2e-test resourceVersion: "22055" uid: 2bcdb93a-a728-456a-8849-2ab1845d2321 kind: Event lastTimestamp: "2026-06-15T06:13:22Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-34378": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-34378": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:13:22Z" name: raw-sklearn-runtime-34378.18b92c73af73bfe9 namespace: kserve-ci-e2e-test resourceVersion: "22080" uid: f9887230-2528-4d9f-80ce-d0b42d054f93 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:14:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-34378 namespace: kserve-ci-e2e-test resourceVersion: "22716" uid: 2bcdb93a-a728-456a-8849-2ab1845d2321 kind: Event lastTimestamp: "2026-06-15T06:14:44Z" message: InferenceService [raw-sklearn-runtime-34378] is Ready metadata: creationTimestamp: "2026-06-15T06:14:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:14:44Z" name: raw-sklearn-runtime-34378.18b92c86c3595972 namespace: kserve-ci-e2e-test resourceVersion: "22717" uid: a8e4281e-be59-4bf7-a2d3-9e770335a4c2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22995"