--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:13:20.260973Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-2519ca-predictor-bc7f75588-pdl67 namespace: kserve-ci-e2e-test resourceVersion: "18947" uid: b2408651-840a-4a52-a4f1-ab7e049dfa2f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-2519ca-predictor-bc7f75588-pdl67 to ip-10-0-139-205.ec2.internal metadata: creationTimestamp: "2026-04-16T20:13:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:13:20Z" name: isvc-init-fail-2519ca-predictor-bc7f75588-pdl67.18a6ef77a9130ff0 namespace: kserve-ci-e2e-test resourceVersion: "18951" uid: 921fa2cb-9271-4599-9eed-a93cbdb5dc0f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58df5c9fbc-5wfsq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-2519ca-predictor-bc7f75588-pdl67 namespace: kserve-ci-e2e-test resourceVersion: "18957" uid: b2408651-840a-4a52-a4f1-ab7e049dfa2f kind: Event lastTimestamp: "2026-04-16T20:13:20Z" message: Add eth0 [10.132.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:13:20Z" name: isvc-init-fail-2519ca-predictor-bc7f75588-pdl67.18a6ef77c3532a5f namespace: kserve-ci-e2e-test resourceVersion: "18968" uid: 186054aa-3dd0-4989-bca8-55a8af8ee785 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-2519ca-predictor-bc7f75588-pdl67 namespace: kserve-ci-e2e-test resourceVersion: "18949" uid: b2408651-840a-4a52-a4f1-ab7e049dfa2f kind: Event lastTimestamp: "2026-04-16T20:13:26Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" already present on machine metadata: creationTimestamp: "2026-04-16T20:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:26Z" name: isvc-init-fail-2519ca-predictor-bc7f75588-pdl67.18a6ef77c4964657 namespace: kserve-ci-e2e-test resourceVersion: "19024" uid: 58f87434-7b01-4d85-ab86-ce1af978e2fc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-2519ca-predictor-bc7f75588-pdl67 namespace: kserve-ci-e2e-test resourceVersion: "18949" uid: b2408651-840a-4a52-a4f1-ab7e049dfa2f kind: Event lastTimestamp: "2026-04-16T20:13:26Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T20:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:26Z" name: isvc-init-fail-2519ca-predictor-bc7f75588-pdl67.18a6ef77c7cad055 namespace: kserve-ci-e2e-test resourceVersion: "19028" uid: 90ef8ddf-4d31-4743-b2f5-4272b88bb0cb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-2519ca-predictor-bc7f75588-pdl67 namespace: kserve-ci-e2e-test resourceVersion: "18949" uid: b2408651-840a-4a52-a4f1-ab7e049dfa2f kind: Event lastTimestamp: "2026-04-16T20:13:26Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T20:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:26Z" name: isvc-init-fail-2519ca-predictor-bc7f75588-pdl67.18a6ef77c8496568 namespace: kserve-ci-e2e-test resourceVersion: "19029" uid: 9ac6cc18-f347-4f50-95cc-ad993d48b744 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-2519ca-predictor-bc7f75588-pdl67 namespace: kserve-ci-e2e-test resourceVersion: "18949" uid: b2408651-840a-4a52-a4f1-ab7e049dfa2f kind: Event lastTimestamp: "2026-04-16T20:13:30Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-04-16T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:30Z" name: isvc-init-fail-2519ca-predictor-bc7f75588-pdl67.18a6ef79fecd0036 namespace: kserve-ci-e2e-test resourceVersion: "19078" uid: c8b6a66f-17be-43d2-ad5f-4a645c561bc5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-2519ca-predictor-bc7f75588 namespace: kserve-ci-e2e-test resourceVersion: "18938" uid: 57d66993-35de-42a0-9b20-70ea503ed6bb kind: Event lastTimestamp: "2026-04-16T20:13:20Z" message: 'Created pod: isvc-init-fail-2519ca-predictor-bc7f75588-pdl67' metadata: creationTimestamp: "2026-04-16T20:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:20Z" name: isvc-init-fail-2519ca-predictor-bc7f75588.18a6ef77a88743e1 namespace: kserve-ci-e2e-test resourceVersion: "18948" uid: e7e50408-e9e0-4783-93e2-29d5050a028b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-2519ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "18936" uid: 08c06680-3f45-4664-a274-70afa66d6a69 kind: Event lastTimestamp: "2026-04-16T20:13:20Z" message: Scaled up replica set isvc-init-fail-2519ca-predictor-bc7f75588 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:20Z" name: isvc-init-fail-2519ca-predictor.18a6ef77a5a37209 namespace: kserve-ci-e2e-test resourceVersion: "18940" uid: 52e80e8a-12fa-4af4-bc54-7079c3f4c075 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-16T20:13:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-2519ca namespace: kserve-ci-e2e-test resourceVersion: "18912" uid: 3bc27a82-ae61-4f3b-9191-a84ab74e1da9 kind: Event lastTimestamp: "2026-04-16T20:13:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:26Z" name: isvc-init-fail-2519ca.18a6ef77a57c4541 namespace: kserve-ci-e2e-test resourceVersion: "19030" uid: 39213aac-e9a1-42bb-a995-212a83041483 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-2519ca namespace: kserve-ci-e2e-test resourceVersion: "18912" uid: 3bc27a82-ae61-4f3b-9191-a84ab74e1da9 kind: Event lastTimestamp: "2026-04-16T20:13:20Z" message: 'Failed to update status for InferenceService "isvc-init-fail-2519ca": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-2519ca": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:20Z" name: isvc-init-fail-2519ca.18a6ef77a5f2dd34 namespace: kserve-ci-e2e-test resourceVersion: "18942" uid: 71704c17-d6ba-4515-b93c-603fb7f222a5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-2519ca namespace: kserve-ci-e2e-test resourceVersion: "18912" uid: 3bc27a82-ae61-4f3b-9191-a84ab74e1da9 kind: Event lastTimestamp: "2026-04-16T20:13:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-2519ca": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:20Z" name: isvc-init-fail-2519ca.18a6ef77a5f3241d namespace: kserve-ci-e2e-test resourceVersion: "18943" uid: a35ae76b-5f80-4825-ab90-bde657141e87 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:07:46.263994Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm namespace: kserve-ci-e2e-test resourceVersion: "16188" uid: a51e5e49-b020-4bca-98aa-7f0c02d441ca kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm to ip-10-0-139-205.ec2.internal metadata: creationTimestamp: "2026-04-16T20:07:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:07:46Z" name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm.18a6ef29e54d9b53 namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: f152e90a-5208-4c32-9700-e6f7d915ac41 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58df5c9fbc-5wfsq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:46Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: a51e5e49-b020-4bca-98aa-7f0c02d441ca kind: Event lastTimestamp: "2026-04-16T20:07:46Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:07:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:07:46Z" name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm.18a6ef29ff415f31 namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: 0ee56799-a190-4d59-9e46-308fefd2900b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: a51e5e49-b020-4bca-98aa-7f0c02d441ca kind: Event lastTimestamp: "2026-04-16T20:07:46Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" already present on machine metadata: creationTimestamp: "2026-04-16T20:07:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:46Z" name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm.18a6ef2a0067b66f namespace: kserve-ci-e2e-test resourceVersion: "16212" uid: c638d69f-820b-4db8-99ab-1d25aa57bc42 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: a51e5e49-b020-4bca-98aa-7f0c02d441ca kind: Event lastTimestamp: "2026-04-16T20:07:46Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T20:07:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:46Z" name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm.18a6ef2a03752c74 namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: 6e459b82-3099-47c0-a5d0-5768ba7e2885 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: a51e5e49-b020-4bca-98aa-7f0c02d441ca kind: Event lastTimestamp: "2026-04-16T20:07:46Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T20:07:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:46Z" name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm.18a6ef2a03e1c150 namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: b2c35028-2d00-4d96-875f-0780c60a60a7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: a51e5e49-b020-4bca-98aa-7f0c02d441ca kind: Event lastTimestamp: "2026-04-16T20:07:51Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:51Z" name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm.18a6ef2b19c0b086 namespace: kserve-ci-e2e-test resourceVersion: "16258" uid: ea1ddae8-b8fd-4ba4-90fc-dd1565cc6607 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: a51e5e49-b020-4bca-98aa-7f0c02d441ca kind: Event lastTimestamp: "2026-04-16T20:07:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:51Z" name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm.18a6ef2b1d3646d9 namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: df2284ee-e345-4e5b-95e3-9ba84187f8f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: a51e5e49-b020-4bca-98aa-7f0c02d441ca kind: Event lastTimestamp: "2026-04-16T20:07:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:51Z" name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm.18a6ef2b1db33f7f namespace: kserve-ci-e2e-test resourceVersion: "16263" uid: 58d3dd8b-3f23-4b1a-b3f1-85359b475eb0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: a51e5e49-b020-4bca-98aa-7f0c02d441ca kind: Event lastTimestamp: "2026-04-16T20:07:51Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:ee4db148793b3f03915a5f82a6e036d7a090e0b557ef078079043d8fab2b15d2" already present on machine metadata: creationTimestamp: "2026-04-16T20:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:51Z" name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm.18a6ef2b1dbffc56 namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: 7a6677e3-4507-4c2f-aa43-929634ff3f3d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: a51e5e49-b020-4bca-98aa-7f0c02d441ca kind: Event lastTimestamp: "2026-04-16T20:07:51Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-16T20:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:51Z" name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm.18a6ef2b21356d82 namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: 89f83bc1-8d82-4d66-ba9c-b8beee268c93 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: a51e5e49-b020-4bca-98aa-7f0c02d441ca kind: Event lastTimestamp: "2026-04-16T20:07:51Z" message: Started container agent metadata: creationTimestamp: "2026-04-16T20:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:51Z" name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm.18a6ef2b21b15c7f namespace: kserve-ci-e2e-test resourceVersion: "16267" uid: 83a0eb80-854e-44d1-9747-4af0f2e00abd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T20:07:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: a51e5e49-b020-4bca-98aa-7f0c02d441ca kind: Event lastTimestamp: "2026-04-16T20:09:34Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:09:34Z" name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm.18a6ef2b55be153b namespace: kserve-ci-e2e-test resourceVersion: "17111" uid: 796487ea-be0f-4dc1-b9ae-7253750a07be reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-16T20:07:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: a51e5e49-b020-4bca-98aa-7f0c02d441ca kind: Event lastTimestamp: "2026-04-16T20:09:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T20:07:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:09:34Z" name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm.18a6ef2b91991bae namespace: kserve-ci-e2e-test resourceVersion: "17112" uid: 2c55ef7c-c2c2-4fec-b130-5cf33ec60fb6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:09:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: a51e5e49-b020-4bca-98aa-7f0c02d441ca kind: Event lastTimestamp: "2026-04-16T20:09:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:09:11Z" name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm.18a6ef3dc1da722a namespace: kserve-ci-e2e-test resourceVersion: "16910" uid: a3b3a1cc-6ebd-446e-80b9-083208680bf1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:09:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: a51e5e49-b020-4bca-98aa-7f0c02d441ca kind: Event lastTimestamp: "2026-04-16T20:09:11Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-16T20:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:09:11Z" name: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm.18a6ef3dc1db7a88 namespace: kserve-ci-e2e-test resourceVersion: "16913" uid: 309d9819-1ed3-45f6-a9d2-ac2e1bf74a83 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-428b1-predictor-754c8b778b namespace: kserve-ci-e2e-test resourceVersion: "16177" uid: 6525e1c5-d356-4aab-a449-6fc8e2d83c54 kind: Event lastTimestamp: "2026-04-16T20:07:46Z" message: 'Created pod: isvc-logger-raw-428b1-predictor-754c8b778b-8vjpm' metadata: creationTimestamp: "2026-04-16T20:07:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:07:46Z" name: isvc-logger-raw-428b1-predictor-754c8b778b.18a6ef29e4e39da6 namespace: kserve-ci-e2e-test resourceVersion: "16189" uid: 5d7156c5-c12b-41d4-8463-26f666b0e54c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-428b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: d25e73a0-6ae4-4da4-aa9a-87dbc3b75f4b kind: Event lastTimestamp: "2026-04-16T20:07:46Z" message: Scaled up replica set isvc-logger-raw-428b1-predictor-754c8b778b from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:07:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:07:46Z" name: isvc-logger-raw-428b1-predictor.18a6ef29e1f8fea6 namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: 25e483e7-f2ef-40f6-99ec-fd6593d2bfbd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:08:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-428b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: 2b2537f7-7a88-4bfb-8a99-445cc3bdd5ff kind: Event lastTimestamp: "2026-04-16T20:08:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:08:31Z" name: isvc-logger-raw-428b1-predictor.18a6ef2d6059ed89 namespace: kserve-ci-e2e-test resourceVersion: "16562" uid: 19df8f56-e917-4622-916b-d18e03273b85 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:08:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-428b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: 2b2537f7-7a88-4bfb-8a99-445cc3bdd5ff kind: Event lastTimestamp: "2026-04-16T20:08:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:08:31Z" name: isvc-logger-raw-428b1-predictor.18a6ef2d60b6c3f7 namespace: kserve-ci-e2e-test resourceVersion: "16563" uid: a6c1c86c-7a6b-4158-aa6b-2c8e9791f0bc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:08:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-428b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16342" uid: 2b2537f7-7a88-4bfb-8a99-445cc3bdd5ff kind: Event lastTimestamp: "2026-04-16T20:09:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:08:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:09:01Z" name: isvc-logger-raw-428b1-predictor.18a6ef37dccede85 namespace: kserve-ci-e2e-test resourceVersion: "16779" uid: 9a996a0b-22b7-4e60-8b90-f231838d9ed8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:08:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-428b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16342" uid: 2b2537f7-7a88-4bfb-8a99-445cc3bdd5ff kind: Event lastTimestamp: "2026-04-16T20:09:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:08:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:09:01Z" name: isvc-logger-raw-428b1-predictor.18a6ef37dd31f374 namespace: kserve-ci-e2e-test resourceVersion: "16780" uid: 47e55f2d-448c-4a6b-992f-d62f8562d193 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T20:07:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-428b1 namespace: kserve-ci-e2e-test resourceVersion: "16167" uid: 845c7c76-0946-4b25-8d01-270898050ff2 kind: Event lastTimestamp: "2026-04-16T20:09:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:07:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:09:04Z" name: isvc-logger-raw-428b1.18a6ef29e1c5af52 namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: b57bf1ac-e25d-4902-b30d-532117f737b2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:07:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-428b1 namespace: kserve-ci-e2e-test resourceVersion: "16167" uid: 845c7c76-0946-4b25-8d01-270898050ff2 kind: Event lastTimestamp: "2026-04-16T20:07:46Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-428b1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-428b1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:07:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:07:46Z" name: isvc-logger-raw-428b1.18a6ef29e224ec10 namespace: kserve-ci-e2e-test resourceVersion: "16198" uid: 57804538-7816-404c-ac96-c01f1c305f0c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-428b1 namespace: kserve-ci-e2e-test resourceVersion: "16175" uid: 845c7c76-0946-4b25-8d01-270898050ff2 kind: Event lastTimestamp: "2026-04-16T20:07:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-428b1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:07:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:07:46Z" name: isvc-logger-raw-428b1.18a6ef29e64219df namespace: kserve-ci-e2e-test resourceVersion: "16199" uid: 8228a859-e004-4ca4-b56c-72508e9e5267 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:09:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-428b1 namespace: kserve-ci-e2e-test resourceVersion: "16807" uid: 845c7c76-0946-4b25-8d01-270898050ff2 kind: Event lastTimestamp: "2026-04-16T20:09:04Z" message: InferenceService [isvc-logger-raw-428b1] is Ready metadata: creationTimestamp: "2026-04-16T20:09:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:09:04Z" name: isvc-logger-raw-428b1.18a6ef3c1ec7e5ae namespace: kserve-ci-e2e-test resourceVersion: "16808" uid: 3360197e-37f0-4ae2-b126-a5ca8b02c48b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:11:41.840687Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn namespace: kserve-ci-e2e-test resourceVersion: "18074" uid: adb6f5cf-cb62-4983-9402-45b83d9ce69b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn to ip-10-0-139-205.ec2.internal metadata: creationTimestamp: "2026-04-16T20:11:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:11:41Z" name: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn.18a6ef60bec4ac91 namespace: kserve-ci-e2e-test resourceVersion: "18080" uid: 74a8ace6-85c1-42e7-94bb-1cb96bbca0ca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58df5c9fbc-5wfsq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:42Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn namespace: kserve-ci-e2e-test resourceVersion: "18083" uid: adb6f5cf-cb62-4983-9402-45b83d9ce69b kind: Event lastTimestamp: "2026-04-16T20:11:42Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:11:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:11:42Z" name: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn.18a6ef60d85d9c16 namespace: kserve-ci-e2e-test resourceVersion: "18091" uid: 843260bd-33d9-445f-b92a-21e977d02afb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn namespace: kserve-ci-e2e-test resourceVersion: "18078" uid: adb6f5cf-cb62-4983-9402-45b83d9ce69b kind: Event lastTimestamp: "2026-04-16T20:11:42Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" already present on machine metadata: creationTimestamp: "2026-04-16T20:11:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:11:42Z" name: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn.18a6ef60d9964b65 namespace: kserve-ci-e2e-test resourceVersion: "18093" uid: 69bcd5bc-41a6-4a86-89c6-989ba01b1aaa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn namespace: kserve-ci-e2e-test resourceVersion: "18078" uid: adb6f5cf-cb62-4983-9402-45b83d9ce69b kind: Event lastTimestamp: "2026-04-16T20:11:42Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T20:11:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:11:42Z" name: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn.18a6ef60dd38e776 namespace: kserve-ci-e2e-test resourceVersion: "18095" uid: bf5711ae-b840-4a45-84bf-423286e88e9e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn namespace: kserve-ci-e2e-test resourceVersion: "18078" uid: adb6f5cf-cb62-4983-9402-45b83d9ce69b kind: Event lastTimestamp: "2026-04-16T20:11:42Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T20:11:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:11:42Z" name: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn.18a6ef60ddb4a5d8 namespace: kserve-ci-e2e-test resourceVersion: "18096" uid: 123d8cf1-2a59-475f-a270-ecf77a9e4ecb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn namespace: kserve-ci-e2e-test resourceVersion: "18078" uid: adb6f5cf-cb62-4983-9402-45b83d9ce69b kind: Event lastTimestamp: "2026-04-16T20:11:47Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:11:47Z" name: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn.18a6ef6205102348 namespace: kserve-ci-e2e-test resourceVersion: "18134" uid: 724725f1-fd6f-4e61-b67d-e15ee150d78c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn namespace: kserve-ci-e2e-test resourceVersion: "18078" uid: adb6f5cf-cb62-4983-9402-45b83d9ce69b kind: Event lastTimestamp: "2026-04-16T20:11:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:11:47Z" name: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn.18a6ef62083a7c19 namespace: kserve-ci-e2e-test resourceVersion: "18139" uid: e28c4ee8-0f3a-46db-b014-8fc15805a161 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn namespace: kserve-ci-e2e-test resourceVersion: "18078" uid: adb6f5cf-cb62-4983-9402-45b83d9ce69b kind: Event lastTimestamp: "2026-04-16T20:11:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:11:47Z" name: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn.18a6ef6208b67c36 namespace: kserve-ci-e2e-test resourceVersion: "18140" uid: d5f68d13-79c7-4949-9bfd-c22b2f5b3fdc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T20:11:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn namespace: kserve-ci-e2e-test resourceVersion: "18078" uid: adb6f5cf-cb62-4983-9402-45b83d9ce69b kind: Event lastTimestamp: "2026-04-16T20:12:49Z" message: 'Readiness probe failed: dial tcp 10.132.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:11:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:12:49Z" name: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn.18a6ef62410184c0 namespace: kserve-ci-e2e-test resourceVersion: "18584" uid: db07f5e6-8f68-4890-959b-9df363e5e10e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn namespace: kserve-ci-e2e-test resourceVersion: "18078" uid: adb6f5cf-cb62-4983-9402-45b83d9ce69b kind: Event lastTimestamp: "2026-04-16T20:13:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:20Z" name: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn.18a6ef77a41df866 namespace: kserve-ci-e2e-test resourceVersion: "18931" uid: 9ea13e17-ef4c-4745-a2af-404e96786a3d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-ea9dbb-predictor-5b757bfb59 namespace: kserve-ci-e2e-test resourceVersion: "18064" uid: 91f77696-54be-4a25-8e37-5cb6097d481d kind: Event lastTimestamp: "2026-04-16T20:11:41Z" message: 'Created pod: isvc-primary-ea9dbb-predictor-5b757bfb59-xw9mn' metadata: creationTimestamp: "2026-04-16T20:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:11:41Z" name: isvc-primary-ea9dbb-predictor-5b757bfb59.18a6ef60be4aa18e namespace: kserve-ci-e2e-test resourceVersion: "18075" uid: 888925f0-e716-4ae7-8387-10f7846941a3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-ea9dbb-predictor namespace: kserve-ci-e2e-test resourceVersion: "18061" uid: ef415647-ae04-4706-abc0-b1793393728f kind: Event lastTimestamp: "2026-04-16T20:11:41Z" message: Scaled up replica set isvc-primary-ea9dbb-predictor-5b757bfb59 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:11:41Z" name: isvc-primary-ea9dbb-predictor.18a6ef60ba69d064 namespace: kserve-ci-e2e-test resourceVersion: "18066" uid: 17ccd38b-7123-4183-bf53-9e162cfd0e4e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-ea9dbb-predictor namespace: kserve-ci-e2e-test resourceVersion: "18063" uid: de003bc7-8c01-4d01-85af-a2b2463b35bd kind: Event lastTimestamp: "2026-04-16T20:11:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:11:56Z" name: isvc-primary-ea9dbb-predictor.18a6ef6439327a9a namespace: kserve-ci-e2e-test resourceVersion: "18216" uid: 6003cb62-0a6e-4aea-85fe-5b2d71535732 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-ea9dbb-predictor namespace: kserve-ci-e2e-test resourceVersion: "18063" uid: de003bc7-8c01-4d01-85af-a2b2463b35bd kind: Event lastTimestamp: "2026-04-16T20:11:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:11:56Z" name: isvc-primary-ea9dbb-predictor.18a6ef64399b3722 namespace: kserve-ci-e2e-test resourceVersion: "18217" uid: 1cac8554-1e0e-4909-9cdb-be698ec4c3e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T20:12:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-ea9dbb-predictor namespace: kserve-ci-e2e-test resourceVersion: "18215" uid: de003bc7-8c01-4d01-85af-a2b2463b35bd kind: Event lastTimestamp: "2026-04-16T20:13:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:11Z" name: isvc-primary-ea9dbb-predictor.18a6ef67b89cd717 namespace: kserve-ci-e2e-test resourceVersion: "18823" uid: c6017bde-bcf9-49aa-a739-fcb1062f1c25 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T20:12:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-ea9dbb-predictor namespace: kserve-ci-e2e-test resourceVersion: "18215" uid: de003bc7-8c01-4d01-85af-a2b2463b35bd kind: Event lastTimestamp: "2026-04-16T20:13:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:11Z" name: isvc-primary-ea9dbb-predictor.18a6ef67b8fdf1da namespace: kserve-ci-e2e-test resourceVersion: "18824" uid: c4246863-65f1-4672-959a-260fb59dc00e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T20:11:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-ea9dbb namespace: kserve-ci-e2e-test resourceVersion: "18038" uid: bf34f1f9-d92a-46b7-b64b-62d9a22e5c58 kind: Event lastTimestamp: "2026-04-16T20:12:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:12:59Z" name: isvc-primary-ea9dbb.18a6ef60ba7007b8 namespace: kserve-ci-e2e-test resourceVersion: "18661" uid: 982ab70c-d9df-4ac7-b7b3-e640b52fe2f2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-ea9dbb namespace: kserve-ci-e2e-test resourceVersion: "18038" uid: bf34f1f9-d92a-46b7-b64b-62d9a22e5c58 kind: Event lastTimestamp: "2026-04-16T20:11:41Z" message: 'Failed to update status for InferenceService "isvc-primary-ea9dbb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-ea9dbb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:11:41Z" name: isvc-primary-ea9dbb.18a6ef60bb00727f namespace: kserve-ci-e2e-test resourceVersion: "18069" uid: 73069dcb-4557-4291-81a0-8f95d1d4f242 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-ea9dbb namespace: kserve-ci-e2e-test resourceVersion: "18038" uid: bf34f1f9-d92a-46b7-b64b-62d9a22e5c58 kind: Event lastTimestamp: "2026-04-16T20:11:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-ea9dbb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:11:41Z" name: isvc-primary-ea9dbb.18a6ef60bb00c1ac namespace: kserve-ci-e2e-test resourceVersion: "18071" uid: 6c741a58-4458-4ec9-bcf5-d81b551d1c4e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-ea9dbb namespace: kserve-ci-e2e-test resourceVersion: "18659" uid: bf34f1f9-d92a-46b7-b64b-62d9a22e5c58 kind: Event lastTimestamp: "2026-04-16T20:12:59Z" message: InferenceService [isvc-primary-ea9dbb] is Ready metadata: creationTimestamp: "2026-04-16T20:12:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:12:59Z" name: isvc-primary-ea9dbb.18a6ef72cdd49d41 namespace: kserve-ci-e2e-test resourceVersion: "18660" uid: c4f49a03-9087-48ca-83b8-95ed9fb895d6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:02:30.396439Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4 namespace: kserve-ci-e2e-test resourceVersion: "13299" uid: 28fd4b7f-3939-4849-9dc5-7f5bf2e6176d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4 to ip-10-0-139-205.ec2.internal metadata: creationTimestamp: "2026-04-16T20:02:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:02:30Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4.18a6eee05a2130eb namespace: kserve-ci-e2e-test resourceVersion: "13303" uid: 9eca199a-d91b-44f4-be35-8cd81d077681 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58df5c9fbc-5wfsq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:30Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4 namespace: kserve-ci-e2e-test resourceVersion: "13310" uid: 28fd4b7f-3939-4849-9dc5-7f5bf2e6176d kind: Event lastTimestamp: "2026-04-16T20:02:30Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:02:30Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4.18a6eee073bb5eb6 namespace: kserve-ci-e2e-test resourceVersion: "13322" uid: 15499ced-f510-45a9-be8d-78593b856e13 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4 namespace: kserve-ci-e2e-test resourceVersion: "13301" uid: 28fd4b7f-3939-4849-9dc5-7f5bf2e6176d kind: Event lastTimestamp: "2026-04-16T20:02:30Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" metadata: creationTimestamp: "2026-04-16T20:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:30Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4.18a6eee0750aca24 namespace: kserve-ci-e2e-test resourceVersion: "13324" uid: be7692d7-df96-4e20-9174-6f1246b3f23b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4 namespace: kserve-ci-e2e-test resourceVersion: "13301" uid: 28fd4b7f-3939-4849-9dc5-7f5bf2e6176d kind: Event lastTimestamp: "2026-04-16T20:02:34Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" in 3.322s (3.322s including waiting). Image size: 299671472 bytes.' metadata: creationTimestamp: "2026-04-16T20:02:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:34Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4.18a6eee13b15923c namespace: kserve-ci-e2e-test resourceVersion: "13348" uid: 69b1fd4c-f16e-40ca-97b0-189e249e51cc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4 namespace: kserve-ci-e2e-test resourceVersion: "13301" uid: 28fd4b7f-3939-4849-9dc5-7f5bf2e6176d kind: Event lastTimestamp: "2026-04-16T20:02:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T20:02:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:34Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4.18a6eee13e7ee977 namespace: kserve-ci-e2e-test resourceVersion: "13349" uid: 1fa7342c-f708-4619-b719-ae5658ec6894 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4 namespace: kserve-ci-e2e-test resourceVersion: "13301" uid: 28fd4b7f-3939-4849-9dc5-7f5bf2e6176d kind: Event lastTimestamp: "2026-04-16T20:02:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T20:02:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:34Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4.18a6eee13f33947d namespace: kserve-ci-e2e-test resourceVersion: "13350" uid: adca7154-80e9-4ddb-a872-ff1a583e95b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4 namespace: kserve-ci-e2e-test resourceVersion: "13301" uid: 28fd4b7f-3939-4849-9dc5-7f5bf2e6176d kind: Event lastTimestamp: "2026-04-16T20:02:39Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1412" metadata: creationTimestamp: "2026-04-16T20:02:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:39Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4.18a6eee2678f461f namespace: kserve-ci-e2e-test resourceVersion: "13387" uid: 74702cc0-795c-4bb1-89fc-513bb03059b5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4 namespace: kserve-ci-e2e-test resourceVersion: "13301" uid: 28fd4b7f-3939-4849-9dc5-7f5bf2e6176d kind: Event lastTimestamp: "2026-04-16T20:02:51Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1412" in 12.531s (12.531s including waiting). Image size: 1560613811 bytes.' metadata: creationTimestamp: "2026-04-16T20:02:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:51Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4.18a6eee552823d09 namespace: kserve-ci-e2e-test resourceVersion: "13481" uid: 7e40b31e-0e57-4ec6-97c0-cf695113aa76 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4 namespace: kserve-ci-e2e-test resourceVersion: "13301" uid: 28fd4b7f-3939-4849-9dc5-7f5bf2e6176d kind: Event lastTimestamp: "2026-04-16T20:02:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:02:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:51Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4.18a6eee5566dab5b namespace: kserve-ci-e2e-test resourceVersion: "13484" uid: f74bd645-2c4a-4a20-9606-cedad17f19f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4 namespace: kserve-ci-e2e-test resourceVersion: "13301" uid: 28fd4b7f-3939-4849-9dc5-7f5bf2e6176d kind: Event lastTimestamp: "2026-04-16T20:02:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:02:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:51Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4.18a6eee556ee6cc9 namespace: kserve-ci-e2e-test resourceVersion: "13485" uid: 03c2273c-06d6-4484-b838-bb8abdbfac9d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4 namespace: kserve-ci-e2e-test resourceVersion: "13301" uid: 28fd4b7f-3939-4849-9dc5-7f5bf2e6176d kind: Event lastTimestamp: "2026-04-16T20:02:51Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:ee4db148793b3f03915a5f82a6e036d7a090e0b557ef078079043d8fab2b15d2" metadata: creationTimestamp: "2026-04-16T20:02:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:51Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4.18a6eee556fd6af7 namespace: kserve-ci-e2e-test resourceVersion: "13486" uid: 33c3f2f6-5615-4ad7-becb-3567b13abd62 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4 namespace: kserve-ci-e2e-test resourceVersion: "13301" uid: 28fd4b7f-3939-4849-9dc5-7f5bf2e6176d kind: Event lastTimestamp: "2026-04-16T20:02:53Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:ee4db148793b3f03915a5f82a6e036d7a090e0b557ef078079043d8fab2b15d2" in 1.783s (1.783s including waiting). Image size: 237897299 bytes.' metadata: creationTimestamp: "2026-04-16T20:02:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:53Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4.18a6eee5c145bc0d namespace: kserve-ci-e2e-test resourceVersion: "13495" uid: 7a13bd66-016a-4182-80be-7957fee732b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4 namespace: kserve-ci-e2e-test resourceVersion: "13301" uid: 28fd4b7f-3939-4849-9dc5-7f5bf2e6176d kind: Event lastTimestamp: "2026-04-16T20:02:53Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-16T20:02:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:53Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4.18a6eee5c4dd94a8 namespace: kserve-ci-e2e-test resourceVersion: "13496" uid: 68caab0f-f4a8-42f0-ace1-5d645e2acb04 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4 namespace: kserve-ci-e2e-test resourceVersion: "13301" uid: 28fd4b7f-3939-4849-9dc5-7f5bf2e6176d kind: Event lastTimestamp: "2026-04-16T20:02:53Z" message: Started container agent metadata: creationTimestamp: "2026-04-16T20:02:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:53Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4.18a6eee5c55a6664 namespace: kserve-ci-e2e-test resourceVersion: "13498" uid: fe6832d7-a8a0-4110-a5ab-923e42389a5e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4 namespace: kserve-ci-e2e-test resourceVersion: "13301" uid: 28fd4b7f-3939-4849-9dc5-7f5bf2e6176d kind: Event lastTimestamp: "2026-04-16T20:04:44Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:44Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4.18a6eee5e9b7abd4 namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: d14795e5-755e-4d82-920e-c3b7583a561b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-16T20:02:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4 namespace: kserve-ci-e2e-test resourceVersion: "13301" uid: 28fd4b7f-3939-4849-9dc5-7f5bf2e6176d kind: Event lastTimestamp: "2026-04-16T20:04:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T20:02:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:44Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4.18a6eee62584d83c namespace: kserve-ci-e2e-test resourceVersion: "14439" uid: 53487ed3-466f-4e47-a1dc-25bd820f56bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4 namespace: kserve-ci-e2e-test resourceVersion: "13301" uid: 28fd4b7f-3939-4849-9dc5-7f5bf2e6176d kind: Event lastTimestamp: "2026-04-16T20:04:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:15Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4.18a6eef8d3f8a59d namespace: kserve-ci-e2e-test resourceVersion: "14116" uid: 1dedbddb-f529-4759-86aa-28cc36f1b152 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4 namespace: kserve-ci-e2e-test resourceVersion: "13301" uid: 28fd4b7f-3939-4849-9dc5-7f5bf2e6176d kind: Event lastTimestamp: "2026-04-16T20:04:15Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-16T20:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:15Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4.18a6eef8d3f9eb20 namespace: kserve-ci-e2e-test resourceVersion: "14119" uid: 6b2e7c91-6edf-4e78-b32a-f7b5af9ee796 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d namespace: kserve-ci-e2e-test resourceVersion: "13288" uid: e8c4fc8a-16ae-40b7-be1a-b77d142536f3 kind: Event lastTimestamp: "2026-04-16T20:02:30Z" message: 'Created pod: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d-xj8t4' metadata: creationTimestamp: "2026-04-16T20:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:30Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d.18a6eee059ae3dc0 namespace: kserve-ci-e2e-test resourceVersion: "13300" uid: e94c83a3-d02f-4d3f-b9ad-1a9380ad91a0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-1ce5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13287" uid: 26efdaf9-9e19-49dc-8718-29f16a937eda kind: Event lastTimestamp: "2026-04-16T20:02:30Z" message: Scaled up replica set isvc-raw-sklearn-batcher-1ce5e-predictor-8bcd69f4d from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:30Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor.18a6eee051c5fe7e namespace: kserve-ci-e2e-test resourceVersion: "13289" uid: d2739a89-0ee4-4473-8d35-8f2670ecb341 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:02:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-1ce5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13294" uid: a01df0b3-3c54-4eab-abac-644d9eb64308 kind: Event lastTimestamp: "2026-04-16T20:03:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:03:30Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor.18a6eee3d725e70c namespace: kserve-ci-e2e-test resourceVersion: "13763" uid: f4ba52a2-a099-4e9c-91a2-88ffe905ad9c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:02:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-1ce5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13294" uid: a01df0b3-3c54-4eab-abac-644d9eb64308 kind: Event lastTimestamp: "2026-04-16T20:03:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:03:30Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor.18a6eee3d7879e36 namespace: kserve-ci-e2e-test resourceVersion: "13764" uid: 04ec4744-7759-4e9e-a5c1-d3bd25172190 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:03:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-1ce5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13432" uid: a01df0b3-3c54-4eab-abac-644d9eb64308 kind: Event lastTimestamp: "2026-04-16T20:04:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:04:15Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor.18a6eef1d3a61e64 namespace: kserve-ci-e2e-test resourceVersion: "14095" uid: 7de15620-688a-442d-8252-068c85ad369c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:03:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-1ce5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13432" uid: a01df0b3-3c54-4eab-abac-644d9eb64308 kind: Event lastTimestamp: "2026-04-16T20:04:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:04:15Z" name: isvc-raw-sklearn-batcher-1ce5e-predictor.18a6eef1d3efaa0b namespace: kserve-ci-e2e-test resourceVersion: "14099" uid: 93df2aef-fd7e-4a9a-84d4-801bf61d609c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T20:02:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-1ce5e namespace: kserve-ci-e2e-test resourceVersion: "13277" uid: 37ce6f6d-82b9-47df-9eb8-dfd366553995 kind: Event lastTimestamp: "2026-04-16T20:04:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:04:04Z" name: isvc-raw-sklearn-batcher-1ce5e.18a6eee057b415ca namespace: kserve-ci-e2e-test resourceVersion: "14011" uid: 3e4a1b48-2702-419f-a711-390be59a1a73 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:02:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-1ce5e namespace: kserve-ci-e2e-test resourceVersion: "13277" uid: 37ce6f6d-82b9-47df-9eb8-dfd366553995 kind: Event lastTimestamp: "2026-04-16T20:02:30Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-1ce5e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-1ce5e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:02:30Z" name: isvc-raw-sklearn-batcher-1ce5e.18a6eee0584b9933 namespace: kserve-ci-e2e-test resourceVersion: "13314" uid: ea7a8197-bc50-44c1-b7d1-fc39cd327dba reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:02:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-1ce5e namespace: kserve-ci-e2e-test resourceVersion: "13277" uid: 37ce6f6d-82b9-47df-9eb8-dfd366553995 kind: Event lastTimestamp: "2026-04-16T20:02:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-1ce5e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:02:30Z" name: isvc-raw-sklearn-batcher-1ce5e.18a6eee0584be568 namespace: kserve-ci-e2e-test resourceVersion: "13315" uid: 3e804386-b5e8-43d6-aaf9-ab5ed48610df reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-1ce5e namespace: kserve-ci-e2e-test resourceVersion: "14009" uid: 37ce6f6d-82b9-47df-9eb8-dfd366553995 kind: Event lastTimestamp: "2026-04-16T20:04:04Z" message: InferenceService [isvc-raw-sklearn-batcher-1ce5e] is Ready metadata: creationTimestamp: "2026-04-16T20:04:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:04:04Z" name: isvc-raw-sklearn-batcher-1ce5e.18a6eef633c5d343 namespace: kserve-ci-e2e-test resourceVersion: "14010" uid: 566645c7-9f75-410a-a35b-2a7e1b6a2954 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:13:02.024260Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-ea9dbb-predictor-647dbdd946-trtm7 namespace: kserve-ci-e2e-test resourceVersion: "18712" uid: d6a3ef60-e456-4952-b71d-0e6c04dc39b4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-ea9dbb-predictor-647dbdd946-trtm7 to ip-10-0-139-205.ec2.internal metadata: creationTimestamp: "2026-04-16T20:13:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:13:02Z" name: isvc-secondary-ea9dbb-predictor-647dbdd946-trtm7.18a6ef736a14df4e namespace: kserve-ci-e2e-test resourceVersion: "18717" uid: 14e893ca-1a21-4826-b604-2785a3695a95 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58df5c9fbc-5wfsq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-ea9dbb-predictor-647dbdd946-trtm7 namespace: kserve-ci-e2e-test resourceVersion: "18721" uid: d6a3ef60-e456-4952-b71d-0e6c04dc39b4 kind: Event lastTimestamp: "2026-04-16T20:13:02Z" message: Add eth0 [10.132.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:13:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:13:02Z" name: isvc-secondary-ea9dbb-predictor-647dbdd946-trtm7.18a6ef7383b1eaaf namespace: kserve-ci-e2e-test resourceVersion: "18732" uid: cd1ae9d3-cd96-422b-838d-e73b0b3fc440 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:13:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-ea9dbb-predictor-647dbdd946-trtm7 namespace: kserve-ci-e2e-test resourceVersion: "18714" uid: d6a3ef60-e456-4952-b71d-0e6c04dc39b4 kind: Event lastTimestamp: "2026-04-16T20:13:08Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" already present on machine metadata: creationTimestamp: "2026-04-16T20:13:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:08Z" name: isvc-secondary-ea9dbb-predictor-647dbdd946-trtm7.18a6ef7385354453 namespace: kserve-ci-e2e-test resourceVersion: "18787" uid: 284e7560-c5e8-4a12-a348-46fb8cfd4112 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:13:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-ea9dbb-predictor-647dbdd946-trtm7 namespace: kserve-ci-e2e-test resourceVersion: "18714" uid: d6a3ef60-e456-4952-b71d-0e6c04dc39b4 kind: Event lastTimestamp: "2026-04-16T20:13:08Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T20:13:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:08Z" name: isvc-secondary-ea9dbb-predictor-647dbdd946-trtm7.18a6ef7388cad425 namespace: kserve-ci-e2e-test resourceVersion: "18791" uid: 255bd509-0688-412b-b586-2bd0af6ad0ca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:13:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-ea9dbb-predictor-647dbdd946-trtm7 namespace: kserve-ci-e2e-test resourceVersion: "18714" uid: d6a3ef60-e456-4952-b71d-0e6c04dc39b4 kind: Event lastTimestamp: "2026-04-16T20:13:08Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T20:13:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:08Z" name: isvc-secondary-ea9dbb-predictor-647dbdd946-trtm7.18a6ef738949f87c namespace: kserve-ci-e2e-test resourceVersion: "18792" uid: df791a43-7d65-46e7-a8cd-e8b92824647a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-ea9dbb-predictor-647dbdd946-trtm7 namespace: kserve-ci-e2e-test resourceVersion: "18714" uid: d6a3ef60-e456-4952-b71d-0e6c04dc39b4 kind: Event lastTimestamp: "2026-04-16T20:13:13Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-ea9dbb-predictor-647dbdd946-trtm7_kserve-ci-e2e-test(d6a3ef60-e456-4952-b71d-0e6c04dc39b4) metadata: creationTimestamp: "2026-04-16T20:13:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:13Z" name: isvc-secondary-ea9dbb-predictor-647dbdd946-trtm7.18a6ef761e6ea4e3 namespace: kserve-ci-e2e-test resourceVersion: "18833" uid: 3064ba66-7529-40fc-9b09-4799c7403cbf reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-ea9dbb-predictor-647dbdd946 namespace: kserve-ci-e2e-test resourceVersion: "18703" uid: cee25527-9531-4975-a8e1-20b2e25c960e kind: Event lastTimestamp: "2026-04-16T20:13:02Z" message: 'Created pod: isvc-secondary-ea9dbb-predictor-647dbdd946-trtm7' metadata: creationTimestamp: "2026-04-16T20:13:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:02Z" name: isvc-secondary-ea9dbb-predictor-647dbdd946.18a6ef7369722af0 namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: 56fb5334-63bb-4e8e-a1b8-29d9002c31a1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-ea9dbb-predictor namespace: kserve-ci-e2e-test resourceVersion: "18701" uid: 003968ff-72da-438c-8501-79cdf47e83d2 kind: Event lastTimestamp: "2026-04-16T20:13:01Z" message: Scaled up replica set isvc-secondary-ea9dbb-predictor-647dbdd946 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:13:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:01Z" name: isvc-secondary-ea9dbb-predictor.18a6ef7365fedbc5 namespace: kserve-ci-e2e-test resourceVersion: "18705" uid: 4803370c-bbef-4007-aa0b-60e44f672641 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-ea9dbb-predictor namespace: kserve-ci-e2e-test resourceVersion: "18702" uid: 5e419988-199e-4a6b-b232-6e7138f145e9 kind: Event lastTimestamp: "2026-04-16T20:13:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:13:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:16Z" name: isvc-secondary-ea9dbb-predictor.18a6ef76e483619f namespace: kserve-ci-e2e-test resourceVersion: "18861" uid: d5459eda-89f0-4364-ad4b-de7db645b82d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-ea9dbb-predictor namespace: kserve-ci-e2e-test resourceVersion: "18702" uid: 5e419988-199e-4a6b-b232-6e7138f145e9 kind: Event lastTimestamp: "2026-04-16T20:13:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:13:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:16Z" name: isvc-secondary-ea9dbb-predictor.18a6ef76e4ea6ef8 namespace: kserve-ci-e2e-test resourceVersion: "18863" uid: dbe8ab01-0ebb-4cd8-b95c-f8a64d2665a7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 15 eventTime: null firstTimestamp: "2026-04-16T20:13:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-ea9dbb namespace: kserve-ci-e2e-test resourceVersion: "18693" uid: fdf543c8-1934-4289-b5be-b32c423fe9a0 kind: Event lastTimestamp: "2026-04-16T20:13:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:13:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:15Z" name: isvc-secondary-ea9dbb.18a6ef7365d4e763 namespace: kserve-ci-e2e-test resourceVersion: "18848" uid: 4402ff74-7590-4e0d-badc-cdf77653d96c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:13:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-ea9dbb namespace: kserve-ci-e2e-test resourceVersion: "18693" uid: fdf543c8-1934-4289-b5be-b32c423fe9a0 kind: Event lastTimestamp: "2026-04-16T20:13:02Z" message: 'Failed to update status for InferenceService "isvc-secondary-ea9dbb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-ea9dbb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:13:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:02Z" name: isvc-secondary-ea9dbb.18a6ef73662dc491 namespace: kserve-ci-e2e-test resourceVersion: "18722" uid: d10e2f32-a06f-4cf6-8043-2b7a39bf84ad reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:13:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-ea9dbb namespace: kserve-ci-e2e-test resourceVersion: "18693" uid: fdf543c8-1934-4289-b5be-b32c423fe9a0 kind: Event lastTimestamp: "2026-04-16T20:13:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-ea9dbb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:13:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:02Z" name: isvc-secondary-ea9dbb.18a6ef73662e0db2 namespace: kserve-ci-e2e-test resourceVersion: "18723" uid: 83dcbe42-8679-45d3-86a8-9ce08ec91593 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:04:15.599334Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf namespace: kserve-ci-e2e-test resourceVersion: "14137" uid: 1d8886c5-d968-46cb-af5e-3c4675d104f7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf to ip-10-0-139-205.ec2.internal metadata: creationTimestamp: "2026-04-16T20:04:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:04:15Z" name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf.18a6eef8d8b61b63 namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: 1f7fdd6e-9755-4be1-ad4a-f61bc50099b8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58df5c9fbc-5wfsq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf namespace: kserve-ci-e2e-test resourceVersion: "14150" uid: 1d8886c5-d968-46cb-af5e-3c4675d104f7 kind: Event lastTimestamp: "2026-04-16T20:04:16Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:04:16Z" name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf.18a6eef8f2f21720 namespace: kserve-ci-e2e-test resourceVersion: "14184" uid: 2c364b53-f8d0-4b86-8eee-b7be6a530183 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf namespace: kserve-ci-e2e-test resourceVersion: "14142" uid: 1d8886c5-d968-46cb-af5e-3c4675d104f7 kind: Event lastTimestamp: "2026-04-16T20:04:16Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" already present on machine metadata: creationTimestamp: "2026-04-16T20:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:16Z" name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf.18a6eef8f41b97e0 namespace: kserve-ci-e2e-test resourceVersion: "14187" uid: 71254475-6caa-4048-b471-e50f31de790a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf namespace: kserve-ci-e2e-test resourceVersion: "14142" uid: 1d8886c5-d968-46cb-af5e-3c4675d104f7 kind: Event lastTimestamp: "2026-04-16T20:04:16Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T20:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:16Z" name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf.18a6eef8f7b4ea70 namespace: kserve-ci-e2e-test resourceVersion: "14195" uid: 96155552-18d8-4c50-b671-18d7348eb9a6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf namespace: kserve-ci-e2e-test resourceVersion: "14142" uid: 1d8886c5-d968-46cb-af5e-3c4675d104f7 kind: Event lastTimestamp: "2026-04-16T20:04:16Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T20:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:16Z" name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf.18a6eef8f841221c namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 0874d3f1-e52d-410f-adcb-1253e2ccf50c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf namespace: kserve-ci-e2e-test resourceVersion: "14142" uid: 1d8886c5-d968-46cb-af5e-3c4675d104f7 kind: Event lastTimestamp: "2026-04-16T20:04:20Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:20Z" name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf.18a6eefa034b66f6 namespace: kserve-ci-e2e-test resourceVersion: "14244" uid: 655c90c5-1f3b-4307-ad20-d17eb359fb80 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf namespace: kserve-ci-e2e-test resourceVersion: "14142" uid: 1d8886c5-d968-46cb-af5e-3c4675d104f7 kind: Event lastTimestamp: "2026-04-16T20:04:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:20Z" name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf.18a6eefa068c8024 namespace: kserve-ci-e2e-test resourceVersion: "14249" uid: f1f445d1-6630-47b8-923c-c72016d47c1a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf namespace: kserve-ci-e2e-test resourceVersion: "14142" uid: 1d8886c5-d968-46cb-af5e-3c4675d104f7 kind: Event lastTimestamp: "2026-04-16T20:04:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:20Z" name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf.18a6eefa070aa7d7 namespace: kserve-ci-e2e-test resourceVersion: "14251" uid: 8dddc00e-b7ae-480c-b2cf-cd7016706eaf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T20:04:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf namespace: kserve-ci-e2e-test resourceVersion: "14142" uid: 1d8886c5-d968-46cb-af5e-3c4675d104f7 kind: Event lastTimestamp: "2026-04-16T20:05:22Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:04:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:05:22Z" name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf.18a6eefa3fc465c9 namespace: kserve-ci-e2e-test resourceVersion: "14733" uid: 6dd1c0cf-5c4f-499a-9d8e-b023d4163ec0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf namespace: kserve-ci-e2e-test resourceVersion: "14142" uid: 1d8886c5-d968-46cb-af5e-3c4675d104f7 kind: Event lastTimestamp: "2026-04-16T20:05:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:05:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:05:55Z" name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf.18a6ef1033cfa2ac namespace: kserve-ci-e2e-test resourceVersion: "15078" uid: 65fa4161-0532-435f-ad06-2b7ae0fd6cd3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd namespace: kserve-ci-e2e-test resourceVersion: "14118" uid: 87c69803-9023-414a-833a-55fb061f406f kind: Event lastTimestamp: "2026-04-16T20:04:15Z" message: 'Created pod: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd-4m4wf' metadata: creationTimestamp: "2026-04-16T20:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:04:15Z" name: isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd.18a6eef8d8668657 namespace: kserve-ci-e2e-test resourceVersion: "14141" uid: 6d9b49ca-b7eb-4bcc-a685-ee73f6a65051 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-c600f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14115" uid: 2ff35350-d689-406b-aa5e-ede3b86bdc9b kind: Event lastTimestamp: "2026-04-16T20:04:15Z" message: Scaled up replica set isvc-sklearn-graph-raw-c600f-predictor-7f8b8cc8fd from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:04:15Z" name: isvc-sklearn-graph-raw-c600f-predictor.18a6eef8d46f5ec5 namespace: kserve-ci-e2e-test resourceVersion: "14120" uid: a934b07f-c28c-4e52-807e-e1f6a2e7a893 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-c600f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14117" uid: 1047113a-8863-49be-80b7-31af37936f22 kind: Event lastTimestamp: "2026-04-16T20:04:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:04:30Z" name: isvc-sklearn-graph-raw-c600f-predictor.18a6eefc5341d7a8 namespace: kserve-ci-e2e-test resourceVersion: "14330" uid: 8d88e5c2-f3e7-452a-98e4-f63e78b2381a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-c600f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14117" uid: 1047113a-8863-49be-80b7-31af37936f22 kind: Event lastTimestamp: "2026-04-16T20:04:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:04:30Z" name: isvc-sklearn-graph-raw-c600f-predictor.18a6eefc539473d9 namespace: kserve-ci-e2e-test resourceVersion: "14331" uid: e45f638d-51dd-4b2c-9761-ca945de0bd39 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T20:04:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-c600f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14329" uid: 1047113a-8863-49be-80b7-31af37936f22 kind: Event lastTimestamp: "2026-04-16T20:05:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:05:45Z" name: isvc-sklearn-graph-raw-c600f-predictor.18a6eeffd2bf5821 namespace: kserve-ci-e2e-test resourceVersion: "14914" uid: 090910ae-f8b9-46e4-874f-4ffb45fe0b74 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T20:04:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-c600f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14329" uid: 1047113a-8863-49be-80b7-31af37936f22 kind: Event lastTimestamp: "2026-04-16T20:05:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:05:45Z" name: isvc-sklearn-graph-raw-c600f-predictor.18a6eeffd325a80f namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: 198f0246-f817-4863-9271-328f4903c682 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T20:04:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-c600f namespace: kserve-ci-e2e-test resourceVersion: "14089" uid: 6b2dc396-0bf6-4f9a-8d49-e02b7167ef2a kind: Event lastTimestamp: "2026-04-16T20:05:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:05:32Z" name: isvc-sklearn-graph-raw-c600f.18a6eef8d47af154 namespace: kserve-ci-e2e-test resourceVersion: "14817" uid: 3153e04a-d4e8-42d2-a51d-82f1c60ccb2f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-c600f namespace: kserve-ci-e2e-test resourceVersion: "14089" uid: 6b2dc396-0bf6-4f9a-8d49-e02b7167ef2a kind: Event lastTimestamp: "2026-04-16T20:04:15Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-c600f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-c600f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:04:15Z" name: isvc-sklearn-graph-raw-c600f.18a6eef8d4e176a5 namespace: kserve-ci-e2e-test resourceVersion: "14123" uid: eff06ba2-40c7-4e7c-83a1-4152ad2ab807 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-c600f namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: 6b2dc396-0bf6-4f9a-8d49-e02b7167ef2a kind: Event lastTimestamp: "2026-04-16T20:05:32Z" message: InferenceService [isvc-sklearn-graph-raw-c600f] is Ready metadata: creationTimestamp: "2026-04-16T20:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:05:32Z" name: isvc-sklearn-graph-raw-c600f.18a6ef0acbda0379 namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: ef593532-4805-4352-b5df-4cabf24651cb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:05:55.946238Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4 namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: 5045f7e4-d5de-4bec-af8c-4e46ccdd0a40 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4 to ip-10-0-139-205.ec2.internal metadata: creationTimestamp: "2026-04-16T20:05:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:05:55Z" name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4.18a6ef1035da3c5c namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: 53d420c2-47ea-400d-857f-51155d301f5e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58df5c9fbc-5wfsq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4 namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: 5045f7e4-d5de-4bec-af8c-4e46ccdd0a40 kind: Event lastTimestamp: "2026-04-16T20:05:56Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:05:56Z" name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4.18a6ef105068ed6f namespace: kserve-ci-e2e-test resourceVersion: "15152" uid: 0358f9e9-c4b9-4d81-ab24-5d9e8fa732b1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4 namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: 5045f7e4-d5de-4bec-af8c-4e46ccdd0a40 kind: Event lastTimestamp: "2026-04-16T20:05:56Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" already present on machine metadata: creationTimestamp: "2026-04-16T20:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:05:56Z" name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4.18a6ef1051e085eb namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: 832d21bc-a04f-401c-983c-9ff2070824f5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4 namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: 5045f7e4-d5de-4bec-af8c-4e46ccdd0a40 kind: Event lastTimestamp: "2026-04-16T20:05:56Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T20:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:05:56Z" name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4.18a6ef1055f663bf namespace: kserve-ci-e2e-test resourceVersion: "15161" uid: 75ffd229-aa6e-4562-b12d-5091a659d43f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4 namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: 5045f7e4-d5de-4bec-af8c-4e46ccdd0a40 kind: Event lastTimestamp: "2026-04-16T20:05:56Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T20:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:05:56Z" name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4.18a6ef10567476e4 namespace: kserve-ci-e2e-test resourceVersion: "15162" uid: d1eab53e-fa4a-4da8-bf9c-3cc86fbc6bc1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:06:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4 namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: 5045f7e4-d5de-4bec-af8c-4e46ccdd0a40 kind: Event lastTimestamp: "2026-04-16T20:06:01Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:06:01Z" name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4.18a6ef1164198850 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: f489fd67-0bed-4289-9575-14242bd0f545 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:06:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4 namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: 5045f7e4-d5de-4bec-af8c-4e46ccdd0a40 kind: Event lastTimestamp: "2026-04-16T20:06:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:06:01Z" name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4.18a6ef1168e2b009 namespace: kserve-ci-e2e-test resourceVersion: "15219" uid: 7b2c036a-dea2-4ba8-a856-7f37edf31f82 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:06:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4 namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: 5045f7e4-d5de-4bec-af8c-4e46ccdd0a40 kind: Event lastTimestamp: "2026-04-16T20:06:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:06:01Z" name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4.18a6ef1169614436 namespace: kserve-ci-e2e-test resourceVersion: "15224" uid: e396d990-617a-470f-93a6-89250d5ca42b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T20:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4 namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: 5045f7e4-d5de-4bec-af8c-4e46ccdd0a40 kind: Event lastTimestamp: "2026-04-16T20:07:03Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:03Z" name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4.18a6ef11a05a1fb3 namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: 627f2857-a003-4944-9396-5ee2eabf7436 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4 namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: 5045f7e4-d5de-4bec-af8c-4e46ccdd0a40 kind: Event lastTimestamp: "2026-04-16T20:07:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:36Z" name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4.18a6ef278dcaa587 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 795c5c1d-9588-4bdb-aa8c-93b6d550dcdc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866 namespace: kserve-ci-e2e-test resourceVersion: "15056" uid: 6872b1f8-0e13-4e0f-82c7-3e3637c1e508 kind: Event lastTimestamp: "2026-04-16T20:05:55Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866-th9q4' metadata: creationTimestamp: "2026-04-16T20:05:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:05:55Z" name: isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866.18a6ef1035645f62 namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: aab4188c-7f3d-455a-a75e-025c2c3b640d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-78192-predictor namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: 7e044b26-6c45-4fda-b785-15e17254d8ec kind: Event lastTimestamp: "2026-04-16T20:05:55Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-78192-predictor-774df4866 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:05:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:05:55Z" name: isvc-sklearn-graph-raw-hpa-78192-predictor.18a6ef1030b5cf64 namespace: kserve-ci-e2e-test resourceVersion: "15058" uid: f893c462-517c-47e0-bacb-74be6ad4311d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:06:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-78192-predictor namespace: kserve-ci-e2e-test resourceVersion: "15055" uid: dea3f9b7-6c0a-4198-9c34-04deabdfe380 kind: Event lastTimestamp: "2026-04-16T20:06:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:06:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:06:55Z" name: isvc-sklearn-graph-raw-hpa-78192-predictor.18a6ef13af8bf338 namespace: kserve-ci-e2e-test resourceVersion: "15631" uid: fcd4979d-b9da-412c-912d-5e71a5aeb80e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:06:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-78192-predictor namespace: kserve-ci-e2e-test resourceVersion: "15055" uid: dea3f9b7-6c0a-4198-9c34-04deabdfe380 kind: Event lastTimestamp: "2026-04-16T20:06:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:06:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:06:55Z" name: isvc-sklearn-graph-raw-hpa-78192-predictor.18a6ef13afedf3d5 namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: 146488dd-6bc5-4cae-b74c-752aec7742ab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:07:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-78192-predictor namespace: kserve-ci-e2e-test resourceVersion: "15303" uid: dea3f9b7-6c0a-4198-9c34-04deabdfe380 kind: Event lastTimestamp: "2026-04-16T20:07:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:07:25Z" name: isvc-sklearn-graph-raw-hpa-78192-predictor.18a6ef21ac47e963 namespace: kserve-ci-e2e-test resourceVersion: "15873" uid: 5dd3bb57-906e-4db3-9280-1598324e9ec9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:07:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-78192-predictor namespace: kserve-ci-e2e-test resourceVersion: "15303" uid: dea3f9b7-6c0a-4198-9c34-04deabdfe380 kind: Event lastTimestamp: "2026-04-16T20:07:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:07:25Z" name: isvc-sklearn-graph-raw-hpa-78192-predictor.18a6ef21acac6265 namespace: kserve-ci-e2e-test resourceVersion: "15876" uid: 8734275c-0418-4e63-8aa1-7847f6499b94 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T20:05:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-78192 namespace: kserve-ci-e2e-test resourceVersion: "15047" uid: b7739d82-1702-45ca-9391-8d5fbc5314e4 kind: Event lastTimestamp: "2026-04-16T20:07:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:05:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:07:13Z" name: isvc-sklearn-graph-raw-hpa-78192.18a6ef1030926049 namespace: kserve-ci-e2e-test resourceVersion: "15776" uid: bf93a243-2c1b-4a27-a064-8e07edd3d6f2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-78192 namespace: kserve-ci-e2e-test resourceVersion: "15774" uid: b7739d82-1702-45ca-9391-8d5fbc5314e4 kind: Event lastTimestamp: "2026-04-16T20:07:13Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-78192] is Ready metadata: creationTimestamp: "2026-04-16T20:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:07:13Z" name: isvc-sklearn-graph-raw-hpa-78192.18a6ef222d3a1005 namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 1b7fd8c3-25c2-4512-96ae-a6fa08dd7cdf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:09:11.581179Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz namespace: kserve-ci-e2e-test resourceVersion: "16909" uid: 5ee81949-ac46-4117-803f-7bddd653884c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz to ip-10-0-139-205.ec2.internal metadata: creationTimestamp: "2026-04-16T20:09:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:09:11Z" name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz.18a6ef3dc29a7d0d namespace: kserve-ci-e2e-test resourceVersion: "16920" uid: c8a155b2-0759-4817-a332-80c696560c77 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58df5c9fbc-5wfsq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:09:12Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz namespace: kserve-ci-e2e-test resourceVersion: "16925" uid: 5ee81949-ac46-4117-803f-7bddd653884c kind: Event lastTimestamp: "2026-04-16T20:09:12Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:09:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:09:12Z" name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz.18a6ef3ddc519255 namespace: kserve-ci-e2e-test resourceVersion: "16935" uid: 824e785e-786b-406a-a437-b335a454619b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:09:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz namespace: kserve-ci-e2e-test resourceVersion: "16912" uid: 5ee81949-ac46-4117-803f-7bddd653884c kind: Event lastTimestamp: "2026-04-16T20:09:12Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" already present on machine metadata: creationTimestamp: "2026-04-16T20:09:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:09:12Z" name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz.18a6ef3ddd85ceeb namespace: kserve-ci-e2e-test resourceVersion: "16937" uid: 41439f79-2e5e-4e3d-8a51-b002ad004a16 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:09:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz namespace: kserve-ci-e2e-test resourceVersion: "16912" uid: 5ee81949-ac46-4117-803f-7bddd653884c kind: Event lastTimestamp: "2026-04-16T20:09:12Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T20:09:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:09:12Z" name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz.18a6ef3de09f2bc0 namespace: kserve-ci-e2e-test resourceVersion: "16940" uid: 5d7fbb30-895b-4890-8b8f-b3c018ac264b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:09:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz namespace: kserve-ci-e2e-test resourceVersion: "16912" uid: 5ee81949-ac46-4117-803f-7bddd653884c kind: Event lastTimestamp: "2026-04-16T20:09:12Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T20:09:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:09:12Z" name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz.18a6ef3de1189519 namespace: kserve-ci-e2e-test resourceVersion: "16941" uid: 145d3cc5-644f-408b-ae91-e4be94a2346f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:09:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz namespace: kserve-ci-e2e-test resourceVersion: "16912" uid: 5ee81949-ac46-4117-803f-7bddd653884c kind: Event lastTimestamp: "2026-04-16T20:09:16Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:09:16Z" name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz.18a6ef3ef73f9b22 namespace: kserve-ci-e2e-test resourceVersion: "16983" uid: d0a997ad-644b-4289-9ae1-06058556c6a8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:09:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz namespace: kserve-ci-e2e-test resourceVersion: "16912" uid: 5ee81949-ac46-4117-803f-7bddd653884c kind: Event lastTimestamp: "2026-04-16T20:09:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:09:16Z" name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz.18a6ef3efa84a3d2 namespace: kserve-ci-e2e-test resourceVersion: "16987" uid: 0e99fff3-68aa-4a69-8c08-7dacb9d1b18b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:09:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz namespace: kserve-ci-e2e-test resourceVersion: "16912" uid: 5ee81949-ac46-4117-803f-7bddd653884c kind: Event lastTimestamp: "2026-04-16T20:09:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:09:16Z" name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz.18a6ef3efb014117 namespace: kserve-ci-e2e-test resourceVersion: "16988" uid: c358f75c-5cbb-4e04-a501-d6cb2a6d1f47 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 17 eventTime: null firstTimestamp: "2026-04-16T20:09:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz namespace: kserve-ci-e2e-test resourceVersion: "16912" uid: 5ee81949-ac46-4117-803f-7bddd653884c kind: Event lastTimestamp: "2026-04-16T20:11:45Z" message: 'Readiness probe failed: dial tcp 10.132.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:09:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:11:45Z" name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz.18a6ef3f332d6834 namespace: kserve-ci-e2e-test resourceVersion: "18118" uid: 9b09558d-1162-47d3-865a-fabd25d81e35 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz namespace: kserve-ci-e2e-test resourceVersion: "16912" uid: 5ee81949-ac46-4117-803f-7bddd653884c kind: Event lastTimestamp: "2026-04-16T20:11:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:11:41Z" name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz.18a6ef60ba1c0d1f namespace: kserve-ci-e2e-test resourceVersion: "18065" uid: 1936d664-2163-427d-9ad7-5639aa9e0875 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:09:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6 namespace: kserve-ci-e2e-test resourceVersion: "16886" uid: 26d914ad-a018-4e8b-a39f-09b3eb10ac73 kind: Event lastTimestamp: "2026-04-16T20:09:11Z" message: 'Created pod: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6-r49gz' metadata: creationTimestamp: "2026-04-16T20:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:09:11Z" name: isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6.18a6ef3dc207dc25 namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: 54155e88-ea9d-4c85-a636-ca7999df723b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:09:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-9f1b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16883" uid: a07cfc76-b6df-42bb-98ec-e34e38a9bb84 kind: Event lastTimestamp: "2026-04-16T20:09:11Z" message: Scaled up replica set isvc-sklearn-scale-raw-9f1b7-predictor-854f955cf6 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:09:11Z" name: isvc-sklearn-scale-raw-9f1b7-predictor.18a6ef3dbeb45d06 namespace: kserve-ci-e2e-test resourceVersion: "16888" uid: ddc2a835-4305-46af-bf62-fb57aa590483 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:09:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-9f1b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16885" uid: 4ec7a9eb-88c2-4304-bf11-6f53f9ce40c1 kind: Event lastTimestamp: "2026-04-16T20:09:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:09:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:09:56Z" name: isvc-sklearn-scale-raw-9f1b7-predictor.18a6ef413d78814e namespace: kserve-ci-e2e-test resourceVersion: "17276" uid: 074ce279-61e3-4c49-afe2-7e7ed3faa498 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:09:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-9f1b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16885" uid: 4ec7a9eb-88c2-4304-bf11-6f53f9ce40c1 kind: Event lastTimestamp: "2026-04-16T20:09:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:09:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:09:56Z" name: isvc-sklearn-scale-raw-9f1b7-predictor.18a6ef413de16dd8 namespace: kserve-ci-e2e-test resourceVersion: "17277" uid: 97308b85-89bf-4b62-a1b6-4f0e67830758 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T20:10:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-9f1b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17061" uid: 4ec7a9eb-88c2-4304-bf11-6f53f9ce40c1 kind: Event lastTimestamp: "2026-04-16T20:11:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:10:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:11:41Z" name: isvc-sklearn-scale-raw-9f1b7-predictor.18a6ef4bbab7b817 namespace: kserve-ci-e2e-test resourceVersion: "18035" uid: 39ef9ec7-1734-4b11-80b7-f2e16d7ba070 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T20:10:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-9f1b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17061" uid: 4ec7a9eb-88c2-4304-bf11-6f53f9ce40c1 kind: Event lastTimestamp: "2026-04-16T20:11:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:10:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:11:41Z" name: isvc-sklearn-scale-raw-9f1b7-predictor.18a6ef4bbb1c5beb namespace: kserve-ci-e2e-test resourceVersion: "18036" uid: ab88a5bf-f498-4f3c-8ac5-0fd5dfb82236 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T20:09:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-9f1b7 namespace: kserve-ci-e2e-test resourceVersion: "16864" uid: 9c915d30-a99a-4f93-8333-8f27f6351e86 kind: Event lastTimestamp: "2026-04-16T20:11:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:11:35Z" name: isvc-sklearn-scale-raw-9f1b7.18a6ef3dbec01328 namespace: kserve-ci-e2e-test resourceVersion: "17986" uid: 49e198c0-3c39-4472-891d-2e64d00b6961 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:09:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-9f1b7 namespace: kserve-ci-e2e-test resourceVersion: "16891" uid: 9c915d30-a99a-4f93-8333-8f27f6351e86 kind: Event lastTimestamp: "2026-04-16T20:09:11Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-9f1b7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-9f1b7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:09:11Z" name: isvc-sklearn-scale-raw-9f1b7.18a6ef3dc2c0c5dc namespace: kserve-ci-e2e-test resourceVersion: "16921" uid: 6417f4b0-76e0-4c3e-aefa-21c9865f251b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-9f1b7 namespace: kserve-ci-e2e-test resourceVersion: "17983" uid: 9c915d30-a99a-4f93-8333-8f27f6351e86 kind: Event lastTimestamp: "2026-04-16T20:11:35Z" message: InferenceService [isvc-sklearn-scale-raw-9f1b7] is Ready metadata: creationTimestamp: "2026-04-16T20:11:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:11:35Z" name: isvc-sklearn-scale-raw-9f1b7.18a6ef5f36322fc8 namespace: kserve-ci-e2e-test resourceVersion: "17984" uid: 4435e4df-71b3-47ee-bef0-caae137ebbe5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:04:15.640203Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7 namespace: kserve-ci-e2e-test resourceVersion: "14152" uid: c7d234ad-23e4-4aa7-93d8-78e8d575e7dd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7 to ip-10-0-139-205.ec2.internal metadata: creationTimestamp: "2026-04-16T20:04:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:04:15Z" name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7.18a6eef8db25957a namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: 2db8f9a6-0de3-4543-8291-b778b6604e3e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58df5c9fbc-5wfsq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7 namespace: kserve-ci-e2e-test resourceVersion: "14166" uid: c7d234ad-23e4-4aa7-93d8-78e8d575e7dd kind: Event lastTimestamp: "2026-04-16T20:04:16Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:04:16Z" name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7.18a6eef8f6031389 namespace: kserve-ci-e2e-test resourceVersion: "14189" uid: eb76d44b-0036-4027-b566-a8adb53bdb65 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7 namespace: kserve-ci-e2e-test resourceVersion: "14157" uid: c7d234ad-23e4-4aa7-93d8-78e8d575e7dd kind: Event lastTimestamp: "2026-04-16T20:04:16Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" already present on machine metadata: creationTimestamp: "2026-04-16T20:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:16Z" name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7.18a6eef8f73894b4 namespace: kserve-ci-e2e-test resourceVersion: "14194" uid: 7e6030b6-3616-4d8b-aef8-739dffba65c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7 namespace: kserve-ci-e2e-test resourceVersion: "14157" uid: c7d234ad-23e4-4aa7-93d8-78e8d575e7dd kind: Event lastTimestamp: "2026-04-16T20:04:16Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T20:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:16Z" name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7.18a6eef8fa784774 namespace: kserve-ci-e2e-test resourceVersion: "14197" uid: 24338f24-a02c-43e8-9466-cefad1b13f53 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7 namespace: kserve-ci-e2e-test resourceVersion: "14157" uid: c7d234ad-23e4-4aa7-93d8-78e8d575e7dd kind: Event lastTimestamp: "2026-04-16T20:04:16Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T20:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:16Z" name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7.18a6eef8faf7fdf6 namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: 23e6eeda-af39-4dbd-ad24-5b26563085c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7 namespace: kserve-ci-e2e-test resourceVersion: "14157" uid: c7d234ad-23e4-4aa7-93d8-78e8d575e7dd kind: Event lastTimestamp: "2026-04-16T20:04:20Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-16T20:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:20Z" name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7.18a6eefa031cc7d5 namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 2fb55214-60e3-4227-8a40-83734c0bce31 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7 namespace: kserve-ci-e2e-test resourceVersion: "14157" uid: c7d234ad-23e4-4aa7-93d8-78e8d575e7dd kind: Event lastTimestamp: "2026-04-16T20:04:39Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.199s (19.199s including waiting). Image size: 1306229499 bytes.' metadata: creationTimestamp: "2026-04-16T20:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:39Z" name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7.18a6eefe7b7dbb25 namespace: kserve-ci-e2e-test resourceVersion: "14397" uid: eaaa5274-9e91-490c-82ff-c4e39c2997ee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7 namespace: kserve-ci-e2e-test resourceVersion: "14157" uid: c7d234ad-23e4-4aa7-93d8-78e8d575e7dd kind: Event lastTimestamp: "2026-04-16T20:04:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:39Z" name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7.18a6eefe7ee5c56b namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: ad98d979-3628-4cb8-a1bc-ac4330a71bd4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7 namespace: kserve-ci-e2e-test resourceVersion: "14157" uid: c7d234ad-23e4-4aa7-93d8-78e8d575e7dd kind: Event lastTimestamp: "2026-04-16T20:04:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:39Z" name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7.18a6eefe7f602831 namespace: kserve-ci-e2e-test resourceVersion: "14399" uid: 8f68df7c-9902-4035-8e10-77e8bdb00294 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T20:04:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7 namespace: kserve-ci-e2e-test resourceVersion: "14157" uid: c7d234ad-23e4-4aa7-93d8-78e8d575e7dd kind: Event lastTimestamp: "2026-04-16T20:05:31Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:05:31Z" name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7.18a6eefeb0e584b0 namespace: kserve-ci-e2e-test resourceVersion: "14798" uid: e2566552-fc1d-452b-947b-ece75c86c228 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7 namespace: kserve-ci-e2e-test resourceVersion: "14157" uid: c7d234ad-23e4-4aa7-93d8-78e8d575e7dd kind: Event lastTimestamp: "2026-04-16T20:05:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:05:56Z" name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7.18a6ef103fc7f6ad namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: 7fba1c5b-1336-40ed-8435-bb810f4a2df9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99 namespace: kserve-ci-e2e-test resourceVersion: "14135" uid: df03d453-7ae5-4ca6-b878-6bcbf36b4ee4 kind: Event lastTimestamp: "2026-04-16T20:04:15Z" message: 'Created pod: isvc-xgboost-graph-raw-c600f-predictor-695c695d99-nbrm7' metadata: creationTimestamp: "2026-04-16T20:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:04:15Z" name: isvc-xgboost-graph-raw-c600f-predictor-695c695d99.18a6eef8dac23b8b namespace: kserve-ci-e2e-test resourceVersion: "14156" uid: 93c9ba6d-f63c-47e4-bc49-328372a59546 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-c600f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14133" uid: 06103b99-1bb4-4213-acf9-b2ff65b42178 kind: Event lastTimestamp: "2026-04-16T20:04:15Z" message: Scaled up replica set isvc-xgboost-graph-raw-c600f-predictor-695c695d99 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:04:15Z" name: isvc-xgboost-graph-raw-c600f-predictor.18a6eef8d826ea33 namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: 182486e8-964c-429d-9dd8-97807c1c73c7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:04:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-c600f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14136" uid: dd910ab5-b5fa-4785-a1d1-f1f6b340086e kind: Event lastTimestamp: "2026-04-16T20:05:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:05:00Z" name: isvc-xgboost-graph-raw-c600f-predictor.18a6eefc56c9070c namespace: kserve-ci-e2e-test resourceVersion: "14564" uid: 89b04934-2d68-48c6-b70a-03edff8c31e7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:04:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-c600f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14136" uid: dd910ab5-b5fa-4785-a1d1-f1f6b340086e kind: Event lastTimestamp: "2026-04-16T20:05:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:05:00Z" name: isvc-xgboost-graph-raw-c600f-predictor.18a6eefc571b40ae namespace: kserve-ci-e2e-test resourceVersion: "14565" uid: c57d36d9-87cd-4ece-aa86-cf3c93b049da reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:05:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-c600f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14332" uid: dd910ab5-b5fa-4785-a1d1-f1f6b340086e kind: Event lastTimestamp: "2026-04-16T20:05:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:05:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:05:45Z" name: isvc-xgboost-graph-raw-c600f-predictor.18a6ef06d34a1478 namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: f4d14c03-691e-4195-9057-682fcc7dbd32 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:05:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-c600f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14332" uid: dd910ab5-b5fa-4785-a1d1-f1f6b340086e kind: Event lastTimestamp: "2026-04-16T20:05:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:05:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:05:45Z" name: isvc-xgboost-graph-raw-c600f-predictor.18a6ef06d3b41ff6 namespace: kserve-ci-e2e-test resourceVersion: "14934" uid: e5b32884-e0e9-44f3-89ea-e69e6112c4b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T20:04:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-c600f namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: acd92ebf-baa7-446e-a2c0-af5835eeffab kind: Event lastTimestamp: "2026-04-16T20:05:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:05:41Z" name: isvc-xgboost-graph-raw-c600f.18a6eef8d836306b namespace: kserve-ci-e2e-test resourceVersion: "14880" uid: 1aa32931-04c4-4a35-bc0d-e04f687591e4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-c600f namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: acd92ebf-baa7-446e-a2c0-af5835eeffab kind: Event lastTimestamp: "2026-04-16T20:04:15Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-c600f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-c600f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:04:15Z" name: isvc-xgboost-graph-raw-c600f.18a6eef8d88e2f2e namespace: kserve-ci-e2e-test resourceVersion: "14144" uid: adaed6b6-4659-4dca-8f31-8fe7d110feab reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-c600f namespace: kserve-ci-e2e-test resourceVersion: "14878" uid: acd92ebf-baa7-446e-a2c0-af5835eeffab kind: Event lastTimestamp: "2026-04-16T20:05:41Z" message: InferenceService [isvc-xgboost-graph-raw-c600f] is Ready metadata: creationTimestamp: "2026-04-16T20:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:05:41Z" name: isvc-xgboost-graph-raw-c600f.18a6ef0ce9337b98 namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: 368c618e-574c-40f5-ac91-e11b852a4b87 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:05:55.998072Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 60663a17-ce51-43a0-b0f3-2faf1d036f53 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h to ip-10-0-139-205.ec2.internal metadata: creationTimestamp: "2026-04-16T20:05:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:05:55Z" name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h.18a6ef1038f1162c namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 0e8c8ed1-b350-4ff2-8923-3972320409a6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58df5c9fbc-5wfsq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 60663a17-ce51-43a0-b0f3-2faf1d036f53 kind: Event lastTimestamp: "2026-04-16T20:05:56Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:05:56Z" name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h.18a6ef105385099a namespace: kserve-ci-e2e-test resourceVersion: "15157" uid: 54056f2a-32c8-4650-b5a6-9e955090d6a6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 60663a17-ce51-43a0-b0f3-2faf1d036f53 kind: Event lastTimestamp: "2026-04-16T20:05:56Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" already present on machine metadata: creationTimestamp: "2026-04-16T20:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:05:56Z" name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h.18a6ef1054a23218 namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 8f2a4e10-472b-420f-9d06-79424b95ede9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 60663a17-ce51-43a0-b0f3-2faf1d036f53 kind: Event lastTimestamp: "2026-04-16T20:05:56Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T20:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:05:56Z" name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h.18a6ef1057fd2b9c namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: ef148442-1dd9-4023-8a9d-a0f8105be65f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 60663a17-ce51-43a0-b0f3-2faf1d036f53 kind: Event lastTimestamp: "2026-04-16T20:05:56Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T20:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:05:56Z" name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h.18a6ef105878295f namespace: kserve-ci-e2e-test resourceVersion: "15164" uid: 39ae2dff-e824-46d0-9507-f6b771d76992 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:06:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 60663a17-ce51-43a0-b0f3-2faf1d036f53 kind: Event lastTimestamp: "2026-04-16T20:06:01Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-16T20:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:06:01Z" name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h.18a6ef11644be3a1 namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: f407e59f-1d32-465f-a939-c3ca955ffaa3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:06:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 60663a17-ce51-43a0-b0f3-2faf1d036f53 kind: Event lastTimestamp: "2026-04-16T20:06:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:06:01Z" name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h.18a6ef1168dc926a namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 74ec135f-a746-4d54-baf4-5842139bfdcc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:06:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 60663a17-ce51-43a0-b0f3-2faf1d036f53 kind: Event lastTimestamp: "2026-04-16T20:06:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:06:01Z" name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h.18a6ef1169549c27 namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 34f60e13-dc81-43fe-bb09-6c2ac119cade reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T20:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 60663a17-ce51-43a0-b0f3-2faf1d036f53 kind: Event lastTimestamp: "2026-04-16T20:06:53Z" message: 'Readiness probe failed: dial tcp 10.132.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:06:53Z" name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h.18a6ef11a07576e9 namespace: kserve-ci-e2e-test resourceVersion: "15615" uid: b2300a62-3bd1-4bb9-b447-562526fdb413 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 60663a17-ce51-43a0-b0f3-2faf1d036f53 kind: Event lastTimestamp: "2026-04-16T20:07:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:36Z" name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h.18a6ef279490a274 namespace: kserve-ci-e2e-test resourceVersion: "16066" uid: 885cba5e-bc4f-404a-bef8-10ade78123da reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: 4c8553d5-af7e-4923-917b-58ee398251b3 kind: Event lastTimestamp: "2026-04-16T20:05:55Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c-l7h2h' metadata: creationTimestamp: "2026-04-16T20:05:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:05:55Z" name: isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c.18a6ef1038776a3f namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: 15b05c33-7035-41a7-b211-fc6ef230e957 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-78192-predictor namespace: kserve-ci-e2e-test resourceVersion: "15087" uid: 76d5f4c4-5ffb-449b-aa38-840fc330fb4e kind: Event lastTimestamp: "2026-04-16T20:05:55Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-78192-predictor-5dd8cfd66c from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:05:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:05:55Z" name: isvc-xgboost-graph-raw-hpa-78192-predictor.18a6ef10360216e1 namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: 1886efeb-2976-4a7e-b554-dfa34a80544d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:06:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-78192-predictor namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: 6ffd72b3-5728-46cd-a46c-8f22f07012be kind: Event lastTimestamp: "2026-04-16T20:06:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:06:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:06:55Z" name: isvc-xgboost-graph-raw-hpa-78192-predictor.18a6ef13b48bd9a8 namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: d697bd03-00e4-4429-925a-74ace7afaf40 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:06:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-78192-predictor namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: 6ffd72b3-5728-46cd-a46c-8f22f07012be kind: Event lastTimestamp: "2026-04-16T20:06:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:06:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:06:56Z" name: isvc-xgboost-graph-raw-hpa-78192-predictor.18a6ef13b4f7e584 namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 9fbfa4dd-3a7d-43f4-b79d-8c5be48a4ce1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:07:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-78192-predictor namespace: kserve-ci-e2e-test resourceVersion: "15306" uid: 6ffd72b3-5728-46cd-a46c-8f22f07012be kind: Event lastTimestamp: "2026-04-16T20:07:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:07:26Z" name: isvc-xgboost-graph-raw-hpa-78192-predictor.18a6ef21affb91cc namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 406bd486-6d71-4f0a-973f-cdd3f6421f78 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:07:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-78192-predictor namespace: kserve-ci-e2e-test resourceVersion: "15306" uid: 6ffd72b3-5728-46cd-a46c-8f22f07012be kind: Event lastTimestamp: "2026-04-16T20:07:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:07:26Z" name: isvc-xgboost-graph-raw-hpa-78192-predictor.18a6ef21b069eaed namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: 8c4cab56-e6bf-4169-a414-dbc4f0f4f34f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T20:05:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-78192 namespace: kserve-ci-e2e-test resourceVersion: "15074" uid: efaceede-523d-4238-b823-39346dbf7e0e kind: Event lastTimestamp: "2026-04-16T20:07:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:05:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:07:03Z" name: isvc-xgboost-graph-raw-hpa-78192.18a6ef1035df467c namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: 2907f48f-9253-40b6-b7af-c196bccecc20 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-78192 namespace: kserve-ci-e2e-test resourceVersion: "15692" uid: efaceede-523d-4238-b823-39346dbf7e0e kind: Event lastTimestamp: "2026-04-16T20:07:03Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-78192] is Ready metadata: creationTimestamp: "2026-04-16T20:07:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:07:03Z" name: isvc-xgboost-graph-raw-hpa-78192.18a6ef1fd89971c1 namespace: kserve-ci-e2e-test resourceVersion: "15693" uid: 25504e15-c0a7-4d43-95fa-d334da63b261 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:07:36.229627Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-428b1-predictor-795678bc55-ssbzv namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 99369020-73f0-4c96-962b-53b691c705c6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-428b1-predictor-795678bc55-ssbzv to ip-10-0-136-138.ec2.internal metadata: creationTimestamp: "2026-04-16T20:07:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:07:36Z" name: message-dumper-raw-428b1-predictor-795678bc55-ssbzv.18a6ef278f3547f5 namespace: kserve-ci-e2e-test resourceVersion: "16039" uid: c9f980f2-1aea-4e0e-aa1d-62e255ffdd96 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58df5c9fbc-5wfsq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:36Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-428b1-predictor-795678bc55-ssbzv namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 99369020-73f0-4c96-962b-53b691c705c6 kind: Event lastTimestamp: "2026-04-16T20:07:36Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:07:36Z" name: message-dumper-raw-428b1-predictor-795678bc55-ssbzv.18a6ef2797069230 namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: 052f45d1-ec96-46c1-9afb-37e3572bac61 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-428b1-predictor-795678bc55-ssbzv namespace: kserve-ci-e2e-test resourceVersion: "16037" uid: 99369020-73f0-4c96-962b-53b691c705c6 kind: Event lastTimestamp: "2026-04-16T20:07:36Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-16T20:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:36Z" name: message-dumper-raw-428b1-predictor-795678bc55-ssbzv.18a6ef2798212fdc namespace: kserve-ci-e2e-test resourceVersion: "16070" uid: 1ccb4dc0-daab-4f5c-9267-ece18a33d5b1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-428b1-predictor-795678bc55-ssbzv namespace: kserve-ci-e2e-test resourceVersion: "16037" uid: 99369020-73f0-4c96-962b-53b691c705c6 kind: Event lastTimestamp: "2026-04-16T20:07:37Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.061s (1.061s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-16T20:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:37Z" name: message-dumper-raw-428b1-predictor-795678bc55-ssbzv.18a6ef27d75f5f8a namespace: kserve-ci-e2e-test resourceVersion: "16083" uid: 590c22de-0b45-4d4d-ba7d-b3a72c906934 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-428b1-predictor-795678bc55-ssbzv namespace: kserve-ci-e2e-test resourceVersion: "16037" uid: 99369020-73f0-4c96-962b-53b691c705c6 kind: Event lastTimestamp: "2026-04-16T20:07:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:37Z" name: message-dumper-raw-428b1-predictor-795678bc55-ssbzv.18a6ef27da2f3784 namespace: kserve-ci-e2e-test resourceVersion: "16084" uid: f90fd1be-136e-4e01-94f3-16d49883e6a8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-428b1-predictor-795678bc55-ssbzv namespace: kserve-ci-e2e-test resourceVersion: "16037" uid: 99369020-73f0-4c96-962b-53b691c705c6 kind: Event lastTimestamp: "2026-04-16T20:07:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:37Z" name: message-dumper-raw-428b1-predictor-795678bc55-ssbzv.18a6ef27da919717 namespace: kserve-ci-e2e-test resourceVersion: "16085" uid: 58c98402-43d5-4d70-81da-c4d57292122c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-428b1-predictor-795678bc55-ssbzv namespace: kserve-ci-e2e-test resourceVersion: "16037" uid: 99369020-73f0-4c96-962b-53b691c705c6 kind: Event lastTimestamp: "2026-04-16T20:07:37Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:37Z" name: message-dumper-raw-428b1-predictor-795678bc55-ssbzv.18a6ef27dca8fc7a namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 6df22bfd-0649-46e8-bb6f-6d0ac4e7b1e2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:09:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-428b1-predictor-795678bc55-ssbzv namespace: kserve-ci-e2e-test resourceVersion: "16037" uid: 99369020-73f0-4c96-962b-53b691c705c6 kind: Event lastTimestamp: "2026-04-16T20:09:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:09:11Z" name: message-dumper-raw-428b1-predictor-795678bc55-ssbzv.18a6ef3dbe2a9880 namespace: kserve-ci-e2e-test resourceVersion: "16884" uid: e50f289b-ea21-439e-85b2-4f889823e85b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-428b1-predictor-795678bc55 namespace: kserve-ci-e2e-test resourceVersion: "16011" uid: 6775f3c1-368a-4c40-9bf3-c82d9331678c kind: Event lastTimestamp: "2026-04-16T20:07:36Z" message: 'Created pod: message-dumper-raw-428b1-predictor-795678bc55-ssbzv' metadata: creationTimestamp: "2026-04-16T20:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:07:36Z" name: message-dumper-raw-428b1-predictor-795678bc55.18a6ef278eb04586 namespace: kserve-ci-e2e-test resourceVersion: "16036" uid: 3a885a03-3228-4a48-98f1-718c647e129b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-428b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16009" uid: 0bba8d8b-aa56-467c-9013-266520f0e384 kind: Event lastTimestamp: "2026-04-16T20:07:36Z" message: Scaled up replica set message-dumper-raw-428b1-predictor-795678bc55 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:07:36Z" name: message-dumper-raw-428b1-predictor.18a6ef278ae2c2fa namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: bd5264e1-6803-4b97-bb7f-9daea9a4c65b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:07:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-428b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16010" uid: 5ab34111-01fa-4ab2-8c82-93b705a6808b kind: Event lastTimestamp: "2026-04-16T20:08:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:08:21Z" name: message-dumper-raw-428b1-predictor.18a6ef2b09715e7f namespace: kserve-ci-e2e-test resourceVersion: "16490" uid: 21a74a65-31bc-4982-bdbc-24b36807e152 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:07:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-428b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16010" uid: 5ab34111-01fa-4ab2-8c82-93b705a6808b kind: Event lastTimestamp: "2026-04-16T20:08:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:08:21Z" name: message-dumper-raw-428b1-predictor.18a6ef2b09dc9f1c namespace: kserve-ci-e2e-test resourceVersion: "16491" uid: e2668b16-7e43-4c01-8655-8687c629d7d6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T20:07:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-428b1 namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 4aa011af-455c-426b-b486-661e473f016c kind: Event lastTimestamp: "2026-04-16T20:07:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:07:38Z" name: message-dumper-raw-428b1.18a6ef278abde9e0 namespace: kserve-ci-e2e-test resourceVersion: "16104" uid: a4a90139-2d47-4c58-8c41-8c38f63f75b8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:07:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-428b1 namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 4aa011af-455c-426b-b486-661e473f016c kind: Event lastTimestamp: "2026-04-16T20:07:36Z" message: 'Failed to update status for InferenceService "message-dumper-raw-428b1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-428b1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:07:36Z" name: message-dumper-raw-428b1.18a6ef278e250041 namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 7475ed61-62f2-4422-acd9-c1808afe89a9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-428b1 namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: 4aa011af-455c-426b-b486-661e473f016c kind: Event lastTimestamp: "2026-04-16T20:07:38Z" message: InferenceService [message-dumper-raw-428b1] is Ready metadata: creationTimestamp: "2026-04-16T20:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:07:38Z" name: message-dumper-raw-428b1.18a6ef281c0e7e9a namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: f635d32f-0414-47d8-a79e-2a6495aa839b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:05:45.676681Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-c600f-6f845876f-ptr82 namespace: kserve-ci-e2e-test resourceVersion: "14923" uid: bf2465e4-3b45-45a9-9501-8c066f2aca25 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-c600f-6f845876f-ptr82 to ip-10-0-139-205.ec2.internal metadata: creationTimestamp: "2026-04-16T20:05:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:05:45Z" name: model-chainer-raw-c600f-6f845876f-ptr82.18a6ef0dd1bd3661 namespace: kserve-ci-e2e-test resourceVersion: "14931" uid: faf0eed3-127c-42e7-abf7-eca8fccf866e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58df5c9fbc-5wfsq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:46Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-c600f-6f845876f-ptr82 namespace: kserve-ci-e2e-test resourceVersion: "14935" uid: bf2465e4-3b45-45a9-9501-8c066f2aca25 kind: Event lastTimestamp: "2026-04-16T20:05:46Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:05:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:05:46Z" name: model-chainer-raw-c600f-6f845876f-ptr82.18a6ef0deb39be29 namespace: kserve-ci-e2e-test resourceVersion: "14946" uid: 8ae111bc-205f-48a0-b779-224327b0a122 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c600f} kind: Pod name: model-chainer-raw-c600f-6f845876f-ptr82 namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: bf2465e4-3b45-45a9-9501-8c066f2aca25 kind: Event lastTimestamp: "2026-04-16T20:05:46Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" metadata: creationTimestamp: "2026-04-16T20:05:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:05:46Z" name: model-chainer-raw-c600f-6f845876f-ptr82.18a6ef0dec89eeed namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: b6e6fb46-3ad2-439b-9f33-f1ab6d6f4ad5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c600f} kind: Pod name: model-chainer-raw-c600f-6f845876f-ptr82 namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: bf2465e4-3b45-45a9-9501-8c066f2aca25 kind: Event lastTimestamp: "2026-04-16T20:05:48Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" in 2.54s (2.54s including waiting). Image size: 216232023 bytes.' metadata: creationTimestamp: "2026-04-16T20:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:05:48Z" name: model-chainer-raw-c600f-6f845876f-ptr82.18a6ef0e83f6a250 namespace: kserve-ci-e2e-test resourceVersion: "14968" uid: 3611228f-326f-407a-9908-3fc95f1254b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c600f} kind: Pod name: model-chainer-raw-c600f-6f845876f-ptr82 namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: bf2465e4-3b45-45a9-9501-8c066f2aca25 kind: Event lastTimestamp: "2026-04-16T20:05:48Z" message: 'Created container: model-chainer-raw-c600f' metadata: creationTimestamp: "2026-04-16T20:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:05:48Z" name: model-chainer-raw-c600f-6f845876f-ptr82.18a6ef0e87be64a9 namespace: kserve-ci-e2e-test resourceVersion: "14969" uid: 16a61657-a4bc-4fab-9513-e22388d5f08e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c600f} kind: Pod name: model-chainer-raw-c600f-6f845876f-ptr82 namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: bf2465e4-3b45-45a9-9501-8c066f2aca25 kind: Event lastTimestamp: "2026-04-16T20:05:48Z" message: Started container model-chainer-raw-c600f metadata: creationTimestamp: "2026-04-16T20:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:05:48Z" name: model-chainer-raw-c600f-6f845876f-ptr82.18a6ef0e883ed12e namespace: kserve-ci-e2e-test resourceVersion: "14970" uid: 1288cd84-30a3-4467-8d12-da88115a65e6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c600f} kind: Pod name: model-chainer-raw-c600f-6f845876f-ptr82 namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: bf2465e4-3b45-45a9-9501-8c066f2aca25 kind: Event lastTimestamp: "2026-04-16T20:05:55Z" message: Stopping container model-chainer-raw-c600f metadata: creationTimestamp: "2026-04-16T20:05:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:05:55Z" name: model-chainer-raw-c600f-6f845876f-ptr82.18a6ef10293e9406 namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: 0d593864-a8ca-423c-94b0-7722b3d116b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:05:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c600f} kind: Pod name: model-chainer-raw-c600f-6f845876f-ptr82 namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: bf2465e4-3b45-45a9-9501-8c066f2aca25 kind: Event lastTimestamp: "2026-04-16T20:06:24Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T20:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:06:24Z" name: model-chainer-raw-c600f-6f845876f-ptr82.18a6ef1125a09d89 namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: c8add721-2054-4093-9579-c6843d838259 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-c600f-6f845876f namespace: kserve-ci-e2e-test resourceVersion: "14920" uid: ef91290e-cae9-4bea-9773-7ca238c85969 kind: Event lastTimestamp: "2026-04-16T20:05:45Z" message: 'Created pod: model-chainer-raw-c600f-6f845876f-ptr82' metadata: creationTimestamp: "2026-04-16T20:05:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:05:45Z" name: model-chainer-raw-c600f-6f845876f.18a6ef0dd117331c namespace: kserve-ci-e2e-test resourceVersion: "14926" uid: 43554583-97bd-462b-bfb5-57a16a244636 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-c600f namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: 056daa3c-e654-47b7-b10e-115d9adaeb3a kind: Event lastTimestamp: "2026-04-16T20:05:45Z" message: Scaled up replica set model-chainer-raw-c600f-6f845876f from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:05:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:05:45Z" name: model-chainer-raw-c600f.18a6ef0dd032f3c8 namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: ecd3ac5b-cd28-47e3-b2a2-1ba756d9a760 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:05:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-c600f namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: 24eff752-eaa7-449d-a7af-182e59015547 kind: Event lastTimestamp: "2026-04-16T20:05:45Z" message: InferenceGraph [model-chainer-raw-c600f] is Ready metadata: creationTimestamp: "2026-04-16T20:05:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:05:45Z" name: model-chainer-raw-c600f.18a6ef0dd4339d96 namespace: kserve-ci-e2e-test resourceVersion: "14939" uid: 806f08a8-d50c-4248-8a5b-e83fd4bad80a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:07:26.002807Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-78192-7f674fdfb9-dx64t namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: 98c6154b-5b56-499a-a050-3300479f7193 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-78192-7f674fdfb9-dx64t to ip-10-0-136-138.ec2.internal metadata: creationTimestamp: "2026-04-16T20:07:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:07:26Z" name: model-chainer-raw-hpa-78192-7f674fdfb9-dx64t.18a6ef252da46817 namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: 32baf64f-a1e8-4217-ab56-27e4a33cb4fb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58df5c9fbc-5wfsq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:26Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-78192-7f674fdfb9-dx64t namespace: kserve-ci-e2e-test resourceVersion: "15886" uid: 98c6154b-5b56-499a-a050-3300479f7193 kind: Event lastTimestamp: "2026-04-16T20:07:26Z" message: Add eth0 [10.134.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:07:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:07:26Z" name: model-chainer-raw-hpa-78192-7f674fdfb9-dx64t.18a6ef2546939b1f namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: fdd7b254-3b83-46e2-bdb0-da24dd69dfaa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-78192} kind: Pod name: model-chainer-raw-hpa-78192-7f674fdfb9-dx64t namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: 98c6154b-5b56-499a-a050-3300479f7193 kind: Event lastTimestamp: "2026-04-16T20:07:26Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" metadata: creationTimestamp: "2026-04-16T20:07:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:26Z" name: model-chainer-raw-hpa-78192-7f674fdfb9-dx64t.18a6ef2547c1fac2 namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: 6bce1e8e-7946-4db3-899e-e6f6c854ee33 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-78192} kind: Pod name: model-chainer-raw-hpa-78192-7f674fdfb9-dx64t namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: 98c6154b-5b56-499a-a050-3300479f7193 kind: Event lastTimestamp: "2026-04-16T20:07:28Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" in 2.372s (2.372s including waiting). Image size: 216232023 bytes.' metadata: creationTimestamp: "2026-04-16T20:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:28Z" name: model-chainer-raw-hpa-78192-7f674fdfb9-dx64t.18a6ef25d5281e4f namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 851a0648-2b89-4a8c-a4f6-75dd332020c2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-78192} kind: Pod name: model-chainer-raw-hpa-78192-7f674fdfb9-dx64t namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: 98c6154b-5b56-499a-a050-3300479f7193 kind: Event lastTimestamp: "2026-04-16T20:07:28Z" message: 'Created container: model-chainer-raw-hpa-78192' metadata: creationTimestamp: "2026-04-16T20:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:28Z" name: model-chainer-raw-hpa-78192-7f674fdfb9-dx64t.18a6ef25d7ca1c86 namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: 0eaad9e1-d546-4051-a5da-2b761b933477 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-78192} kind: Pod name: model-chainer-raw-hpa-78192-7f674fdfb9-dx64t namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: 98c6154b-5b56-499a-a050-3300479f7193 kind: Event lastTimestamp: "2026-04-16T20:07:28Z" message: Started container model-chainer-raw-hpa-78192 metadata: creationTimestamp: "2026-04-16T20:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:28Z" name: model-chainer-raw-hpa-78192-7f674fdfb9-dx64t.18a6ef25d82bb4bf namespace: kserve-ci-e2e-test resourceVersion: "15924" uid: 7e3c7849-726e-497e-97c5-9d6390846c56 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-78192} kind: Pod name: model-chainer-raw-hpa-78192-7f674fdfb9-dx64t namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: 98c6154b-5b56-499a-a050-3300479f7193 kind: Event lastTimestamp: "2026-04-16T20:07:36Z" message: Stopping container model-chainer-raw-hpa-78192 metadata: creationTimestamp: "2026-04-16T20:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:07:36Z" name: model-chainer-raw-hpa-78192-7f674fdfb9-dx64t.18a6ef27853ed03e namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: 446b83cb-faa8-4b2a-8de8-4d870d76e1f3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:07:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-78192} kind: Pod name: model-chainer-raw-hpa-78192-7f674fdfb9-dx64t namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: 98c6154b-5b56-499a-a050-3300479f7193 kind: Event lastTimestamp: "2026-04-16T20:08:05Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T20:07:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:08:05Z" name: model-chainer-raw-hpa-78192-7f674fdfb9-dx64t.18a6ef288e738632 namespace: kserve-ci-e2e-test resourceVersion: "16377" uid: e614eca0-6514-4d9e-8823-700330d400f5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-138.ec2.internal source: component: kubelet host: ip-10-0-136-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-78192-7f674fdfb9 namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: cc11ab00-6b74-4c9c-95cd-3c078fdf3f13 kind: Event lastTimestamp: "2026-04-16T20:07:25Z" message: 'Created pod: model-chainer-raw-hpa-78192-7f674fdfb9-dx64t' metadata: creationTimestamp: "2026-04-16T20:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:07:25Z" name: model-chainer-raw-hpa-78192-7f674fdfb9.18a6ef252d299a3b namespace: kserve-ci-e2e-test resourceVersion: "15879" uid: 67b10537-2c08-41da-be26-ffeba48eb263 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-78192 namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: deccd330-2cb6-4e07-9c21-faf52b9303cb kind: Event lastTimestamp: "2026-04-16T20:07:25Z" message: Scaled up replica set model-chainer-raw-hpa-78192-7f674fdfb9 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:07:25Z" name: model-chainer-raw-hpa-78192.18a6ef252c305fbb namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: b4589ff8-043c-4efc-ad7b-5b90b7a85585 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:07:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-78192 namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: 8f419f1e-a58c-4dc3-ab49-3b061cde8e27 kind: Event lastTimestamp: "2026-04-16T20:07:26Z" message: InferenceGraph [model-chainer-raw-hpa-78192] is Ready metadata: creationTimestamp: "2026-04-16T20:07:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:07:26Z" name: model-chainer-raw-hpa-78192.18a6ef252fe76f8b namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: 934cbbb5-c1ce-49cc-861f-fa10506f2b81 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:13:30.389722Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-05c63-predictor-695786cf5d-6qhgw namespace: kserve-ci-e2e-test resourceVersion: "19098" uid: ab312f2b-0b40-4fbe-871b-7124abc8a68d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-05c63-predictor-695786cf5d-6qhgw to ip-10-0-139-205.ec2.internal metadata: creationTimestamp: "2026-04-16T20:13:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:13:30Z" name: raw-sklearn-05c63-predictor-695786cf5d-6qhgw.18a6ef7a04cb7d5d namespace: kserve-ci-e2e-test resourceVersion: "19103" uid: 963bab47-748d-4be1-8b9a-99dc6845a9bb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58df5c9fbc-5wfsq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:30Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-05c63-predictor-695786cf5d-6qhgw namespace: kserve-ci-e2e-test resourceVersion: "19109" uid: ab312f2b-0b40-4fbe-871b-7124abc8a68d kind: Event lastTimestamp: "2026-04-16T20:13:30Z" message: Add eth0 [10.132.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:13:30Z" name: raw-sklearn-05c63-predictor-695786cf5d-6qhgw.18a6ef7a204b0c4a namespace: kserve-ci-e2e-test resourceVersion: "19124" uid: 2fac40cc-ee85-4c77-b7f2-9a058f9d365c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-05c63-predictor-695786cf5d-6qhgw namespace: kserve-ci-e2e-test resourceVersion: "19100" uid: ab312f2b-0b40-4fbe-871b-7124abc8a68d kind: Event lastTimestamp: "2026-04-16T20:13:30Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" already present on machine metadata: creationTimestamp: "2026-04-16T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:30Z" name: raw-sklearn-05c63-predictor-695786cf5d-6qhgw.18a6ef7a2197c3a1 namespace: kserve-ci-e2e-test resourceVersion: "19127" uid: 8fe769e7-7d0e-4b4f-a688-b2115364315d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-05c63-predictor-695786cf5d-6qhgw namespace: kserve-ci-e2e-test resourceVersion: "19100" uid: ab312f2b-0b40-4fbe-871b-7124abc8a68d kind: Event lastTimestamp: "2026-04-16T20:13:30Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:30Z" name: raw-sklearn-05c63-predictor-695786cf5d-6qhgw.18a6ef7a248165b2 namespace: kserve-ci-e2e-test resourceVersion: "19129" uid: 163b282d-7831-4c3f-a36f-b788fa3566b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-05c63-predictor-695786cf5d-6qhgw namespace: kserve-ci-e2e-test resourceVersion: "19100" uid: ab312f2b-0b40-4fbe-871b-7124abc8a68d kind: Event lastTimestamp: "2026-04-16T20:13:30Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:30Z" name: raw-sklearn-05c63-predictor-695786cf5d-6qhgw.18a6ef7a24fdfca3 namespace: kserve-ci-e2e-test resourceVersion: "19130" uid: 928064c6-a36a-423c-b3ca-40569f09ea03 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-05c63-predictor-695786cf5d-6qhgw namespace: kserve-ci-e2e-test resourceVersion: "19100" uid: ab312f2b-0b40-4fbe-871b-7124abc8a68d kind: Event lastTimestamp: "2026-04-16T20:13:34Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:34Z" name: raw-sklearn-05c63-predictor-695786cf5d-6qhgw.18a6ef7b06ffdec9 namespace: kserve-ci-e2e-test resourceVersion: "19166" uid: 91825d78-234a-4fbe-ac5e-d6466c22bfcd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-05c63-predictor-695786cf5d-6qhgw namespace: kserve-ci-e2e-test resourceVersion: "19100" uid: ab312f2b-0b40-4fbe-871b-7124abc8a68d kind: Event lastTimestamp: "2026-04-16T20:13:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:34Z" name: raw-sklearn-05c63-predictor-695786cf5d-6qhgw.18a6ef7b0a1c4e8c namespace: kserve-ci-e2e-test resourceVersion: "19170" uid: afa808b7-db69-42a4-9878-1e9482030b8b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-05c63-predictor-695786cf5d-6qhgw namespace: kserve-ci-e2e-test resourceVersion: "19100" uid: ab312f2b-0b40-4fbe-871b-7124abc8a68d kind: Event lastTimestamp: "2026-04-16T20:13:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:34Z" name: raw-sklearn-05c63-predictor-695786cf5d-6qhgw.18a6ef7b0a9923bc namespace: kserve-ci-e2e-test resourceVersion: "19171" uid: de4ce015-1509-40b7-a507-ed92aad752e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T20:13:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-05c63-predictor-695786cf5d-6qhgw namespace: kserve-ci-e2e-test resourceVersion: "19100" uid: ab312f2b-0b40-4fbe-871b-7124abc8a68d kind: Event lastTimestamp: "2026-04-16T20:14:37Z" message: 'Readiness probe failed: dial tcp 10.132.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:14:37Z" name: raw-sklearn-05c63-predictor-695786cf5d-6qhgw.18a6ef7b43020eea namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 4082107d-ce6e-47c4-aadf-35632ff7ff0a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-05c63-predictor-695786cf5d-6qhgw namespace: kserve-ci-e2e-test resourceVersion: "19100" uid: ab312f2b-0b40-4fbe-871b-7124abc8a68d kind: Event lastTimestamp: "2026-04-16T20:14:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:14:50Z" name: raw-sklearn-05c63-predictor-695786cf5d-6qhgw.18a6ef8cac2f54b2 namespace: kserve-ci-e2e-test resourceVersion: "19739" uid: 69b78842-3138-483a-8fd5-41a49ad00f45 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-05c63-predictor-695786cf5d namespace: kserve-ci-e2e-test resourceVersion: "19090" uid: a0e76d3f-9898-4058-8a26-0ae05e5c073d kind: Event lastTimestamp: "2026-04-16T20:13:30Z" message: 'Created pod: raw-sklearn-05c63-predictor-695786cf5d-6qhgw' metadata: creationTimestamp: "2026-04-16T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:30Z" name: raw-sklearn-05c63-predictor-695786cf5d.18a6ef7a045e9f80 namespace: kserve-ci-e2e-test resourceVersion: "19099" uid: 3a49ae52-8831-4c40-9abe-4127b84ad9ac reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-05c63-predictor namespace: kserve-ci-e2e-test resourceVersion: "19088" uid: 27b044c3-6b71-40bd-a0c6-1e5928979cfb kind: Event lastTimestamp: "2026-04-16T20:13:30Z" message: Scaled up replica set raw-sklearn-05c63-predictor-695786cf5d from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:30Z" name: raw-sklearn-05c63-predictor.18a6ef7a020c326a namespace: kserve-ci-e2e-test resourceVersion: "19092" uid: b238fdbc-510f-43ce-a60d-51190d76838c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:13:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-05c63-predictor namespace: kserve-ci-e2e-test resourceVersion: "19089" uid: 811f7d1d-62d6-45fa-a310-b19c212e0d20 kind: Event lastTimestamp: "2026-04-16T20:14:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:15Z" name: raw-sklearn-05c63-predictor.18a6ef7d80e2ac15 namespace: kserve-ci-e2e-test resourceVersion: "19457" uid: 66345888-cb19-4a81-ba40-62fba0cf8c11 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:13:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-05c63-predictor namespace: kserve-ci-e2e-test resourceVersion: "19089" uid: 811f7d1d-62d6-45fa-a310-b19c212e0d20 kind: Event lastTimestamp: "2026-04-16T20:14:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:15Z" name: raw-sklearn-05c63-predictor.18a6ef7d81402eef namespace: kserve-ci-e2e-test resourceVersion: "19458" uid: e0c848df-8fad-47a3-b9ef-20c5a0191d6f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:14:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-05c63-predictor namespace: kserve-ci-e2e-test resourceVersion: "19248" uid: 811f7d1d-62d6-45fa-a310-b19c212e0d20 kind: Event lastTimestamp: "2026-04-16T20:14:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:45Z" name: raw-sklearn-05c63-predictor.18a6ef87fe11a013 namespace: kserve-ci-e2e-test resourceVersion: "19665" uid: d1e944df-8680-44d6-b195-b8863318a9c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:14:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-05c63-predictor namespace: kserve-ci-e2e-test resourceVersion: "19248" uid: 811f7d1d-62d6-45fa-a310-b19c212e0d20 kind: Event lastTimestamp: "2026-04-16T20:14:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:45Z" name: raw-sklearn-05c63-predictor.18a6ef87fe7e6843 namespace: kserve-ci-e2e-test resourceVersion: "19666" uid: 48ca35db-0804-4f62-9bc6-fcdcfe065746 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T20:13:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-05c63 namespace: kserve-ci-e2e-test resourceVersion: "19079" uid: 3a00d73a-365c-416e-ba2a-f26d49541d34 kind: Event lastTimestamp: "2026-04-16T20:14:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:14:47Z" name: raw-sklearn-05c63.18a6ef7a01edc6a7 namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: 33cb5c72-07b9-474e-b6db-413af919900f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:13:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-05c63 namespace: kserve-ci-e2e-test resourceVersion: "19079" uid: 3a00d73a-365c-416e-ba2a-f26d49541d34 kind: Event lastTimestamp: "2026-04-16T20:13:30Z" message: 'Failed to update status for InferenceService "raw-sklearn-05c63": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-05c63": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:30Z" name: raw-sklearn-05c63.18a6ef7a02a8872b namespace: kserve-ci-e2e-test resourceVersion: "19114" uid: c0145461-c7d5-4b01-b02e-cb523e8a4534 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-05c63 namespace: kserve-ci-e2e-test resourceVersion: "19096" uid: 3a00d73a-365c-416e-ba2a-f26d49541d34 kind: Event lastTimestamp: "2026-04-16T20:13:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-05c63": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:30Z" name: raw-sklearn-05c63.18a6ef7a08fd2cb6 namespace: kserve-ci-e2e-test resourceVersion: "19116" uid: ddc771a9-d7a9-42a7-be1f-7d24e4765abe reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-05c63 namespace: kserve-ci-e2e-test resourceVersion: "19686" uid: 3a00d73a-365c-416e-ba2a-f26d49541d34 kind: Event lastTimestamp: "2026-04-16T20:14:47Z" message: InferenceService [raw-sklearn-05c63] is Ready metadata: creationTimestamp: "2026-04-16T20:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:14:47Z" name: raw-sklearn-05c63.18a6ef8beb63c41c namespace: kserve-ci-e2e-test resourceVersion: "19688" uid: e864a2a5-9d38-4600-ac79-0fb5688dab51 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:14:50.579619Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7 namespace: kserve-ci-e2e-test resourceVersion: "19754" uid: 87d01517-0f0c-4405-9c3f-7d59fc59b1ab kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7 to ip-10-0-139-205.ec2.internal metadata: creationTimestamp: "2026-04-16T20:14:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:14:50Z" name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7.18a6ef8cb07c2297 namespace: kserve-ci-e2e-test resourceVersion: "19758" uid: ae7aa4c1-caba-4af1-a14c-5d3fe8751cc0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58df5c9fbc-5wfsq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:51Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7 namespace: kserve-ci-e2e-test resourceVersion: "19763" uid: 87d01517-0f0c-4405-9c3f-7d59fc59b1ab kind: Event lastTimestamp: "2026-04-16T20:14:51Z" message: Add eth0 [10.132.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:14:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:14:51Z" name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7.18a6ef8cca2d1948 namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: 60f0dab8-6fb8-41db-8b44-8287a7bc2e1e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7 namespace: kserve-ci-e2e-test resourceVersion: "19756" uid: 87d01517-0f0c-4405-9c3f-7d59fc59b1ab kind: Event lastTimestamp: "2026-04-16T20:14:51Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" already present on machine metadata: creationTimestamp: "2026-04-16T20:14:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:14:51Z" name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7.18a6ef8ccb74b828 namespace: kserve-ci-e2e-test resourceVersion: "19774" uid: 457622c1-3b25-416d-a4ee-16de715ec94b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7 namespace: kserve-ci-e2e-test resourceVersion: "19756" uid: 87d01517-0f0c-4405-9c3f-7d59fc59b1ab kind: Event lastTimestamp: "2026-04-16T20:14:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T20:14:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:14:51Z" name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7.18a6ef8ccf0beb78 namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 3bc63b7f-c020-4587-912b-e550465d6180 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7 namespace: kserve-ci-e2e-test resourceVersion: "19756" uid: 87d01517-0f0c-4405-9c3f-7d59fc59b1ab kind: Event lastTimestamp: "2026-04-16T20:14:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T20:14:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:14:51Z" name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7.18a6ef8ccf81aa91 namespace: kserve-ci-e2e-test resourceVersion: "19779" uid: 062ce287-fcc2-4ced-8192-ccf9627a258d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7 namespace: kserve-ci-e2e-test resourceVersion: "19756" uid: 87d01517-0f0c-4405-9c3f-7d59fc59b1ab kind: Event lastTimestamp: "2026-04-16T20:14:55Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:14:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:14:55Z" name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7.18a6ef8db8e60a51 namespace: kserve-ci-e2e-test resourceVersion: "19817" uid: 7a7f5c4b-2a1c-480b-8fb6-771691a951c7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7 namespace: kserve-ci-e2e-test resourceVersion: "19756" uid: 87d01517-0f0c-4405-9c3f-7d59fc59b1ab kind: Event lastTimestamp: "2026-04-16T20:14:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:14:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:14:55Z" name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7.18a6ef8dbc35bc03 namespace: kserve-ci-e2e-test resourceVersion: "19821" uid: 2512d1ce-c8ae-4927-b16d-d19e6a6495a3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7 namespace: kserve-ci-e2e-test resourceVersion: "19756" uid: 87d01517-0f0c-4405-9c3f-7d59fc59b1ab kind: Event lastTimestamp: "2026-04-16T20:14:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:14:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:14:55Z" name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7.18a6ef8dbcbffe60 namespace: kserve-ci-e2e-test resourceVersion: "19822" uid: 1f28661a-d98d-405c-bc82-5044525c023c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T20:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7 namespace: kserve-ci-e2e-test resourceVersion: "19756" uid: 87d01517-0f0c-4405-9c3f-7d59fc59b1ab kind: Event lastTimestamp: "2026-04-16T20:15:57Z" message: 'Readiness probe failed: dial tcp 10.132.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:15:57Z" name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7.18a6ef8df4da9993 namespace: kserve-ci-e2e-test resourceVersion: "20266" uid: ca6151ca-6ce0-42b3-963a-969e9f6984dd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:16:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7 namespace: kserve-ci-e2e-test resourceVersion: "19756" uid: 87d01517-0f0c-4405-9c3f-7d59fc59b1ab kind: Event lastTimestamp: "2026-04-16T20:16:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:16:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:16:10Z" name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7.18a6ef9f598032b1 namespace: kserve-ci-e2e-test resourceVersion: "20382" uid: 254b292e-90ab-4c61-a74a-d06279ed1825 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-205.ec2.internal source: component: kubelet host: ip-10-0-139-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb namespace: kserve-ci-e2e-test resourceVersion: "19744" uid: 3948a050-15f2-48ec-8b68-336a040e1164 kind: Event lastTimestamp: "2026-04-16T20:14:50Z" message: 'Created pod: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb-5hvz7' metadata: creationTimestamp: "2026-04-16T20:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:50Z" name: raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb.18a6ef8cb0123681 namespace: kserve-ci-e2e-test resourceVersion: "19755" uid: 80bcc261-5845-4bbf-a133-2ca917e2f227 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-a6d09-predictor namespace: kserve-ci-e2e-test resourceVersion: "19742" uid: c670c8d6-2599-425e-ba11-3141c01c9445 kind: Event lastTimestamp: "2026-04-16T20:14:50Z" message: Scaled up replica set raw-sklearn-runtime-a6d09-predictor-7dfcbcc5bb from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:50Z" name: raw-sklearn-runtime-a6d09-predictor.18a6ef8cad488237 namespace: kserve-ci-e2e-test resourceVersion: "19746" uid: 1395bf21-a707-4a47-ac92-4ae92c7faf74 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:15:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-a6d09-predictor namespace: kserve-ci-e2e-test resourceVersion: "19743" uid: 6d0711c5-2c27-45ee-9947-19be485bd218 kind: Event lastTimestamp: "2026-04-16T20:15:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:15:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:15:35Z" name: raw-sklearn-runtime-a6d09-predictor.18a6ef902c181a4b namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: db510eba-67b6-450b-80f5-2ab29f394840 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:15:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-a6d09-predictor namespace: kserve-ci-e2e-test resourceVersion: "19743" uid: 6d0711c5-2c27-45ee-9947-19be485bd218 kind: Event lastTimestamp: "2026-04-16T20:15:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:15:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:15:35Z" name: raw-sklearn-runtime-a6d09-predictor.18a6ef902c7fa04d namespace: kserve-ci-e2e-test resourceVersion: "20113" uid: 710299f5-68c0-43a2-995a-f0abb3bd0a2f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:15:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-a6d09-predictor namespace: kserve-ci-e2e-test resourceVersion: "19903" uid: 6d0711c5-2c27-45ee-9947-19be485bd218 kind: Event lastTimestamp: "2026-04-16T20:16:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:15:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:16:05Z" name: raw-sklearn-runtime-a6d09-predictor.18a6ef9aa95ca223 namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: 96514aa8-c82a-4d8b-9b65-4d8afb4af26e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:15:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-a6d09-predictor namespace: kserve-ci-e2e-test resourceVersion: "19903" uid: 6d0711c5-2c27-45ee-9947-19be485bd218 kind: Event lastTimestamp: "2026-04-16T20:16:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:15:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:16:05Z" name: raw-sklearn-runtime-a6d09-predictor.18a6ef9aa9bd2728 namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: 0a7d7254-810e-4572-acc1-d549dd8fcc18 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-04-16T20:14:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-a6d09 namespace: kserve-ci-e2e-test resourceVersion: "19714" uid: 96f1fc0b-7c6f-4c28-873d-60524f16f09b kind: Event lastTimestamp: "2026-04-16T20:16:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:16:07Z" name: raw-sklearn-runtime-a6d09.18a6ef8cad2501b6 namespace: kserve-ci-e2e-test resourceVersion: "20340" uid: dfe0c808-688d-4490-8863-2714bb8cf8f0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:14:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-a6d09 namespace: kserve-ci-e2e-test resourceVersion: "19714" uid: 96f1fc0b-7c6f-4c28-873d-60524f16f09b kind: Event lastTimestamp: "2026-04-16T20:14:50Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-a6d09": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-a6d09": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:14:50Z" name: raw-sklearn-runtime-a6d09.18a6ef8cadd02716 namespace: kserve-ci-e2e-test resourceVersion: "19765" uid: c32051a1-0a5f-429d-a016-e3589d06e0b9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-a6d09 namespace: kserve-ci-e2e-test resourceVersion: "19725" uid: 96f1fc0b-7c6f-4c28-873d-60524f16f09b kind: Event lastTimestamp: "2026-04-16T20:14:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-a6d09": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:14:50Z" name: raw-sklearn-runtime-a6d09.18a6ef8cb2a447ec namespace: kserve-ci-e2e-test resourceVersion: "19766" uid: 77812a88-7ba2-41d7-8c9e-4d1c67f89131 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:16:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-a6d09 namespace: kserve-ci-e2e-test resourceVersion: "20338" uid: 96f1fc0b-7c6f-4c28-873d-60524f16f09b kind: Event lastTimestamp: "2026-04-16T20:16:07Z" message: InferenceService [raw-sklearn-runtime-a6d09] is Ready metadata: creationTimestamp: "2026-04-16T20:16:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:16:07Z" name: raw-sklearn-runtime-a6d09.18a6ef9e8108c397 namespace: kserve-ci-e2e-test resourceVersion: "20339" uid: e79359d8-7703-4176-bd28-97853cec4b2c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "20568"