--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:44:46.697617Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-21c95a-predictor-56c84f677d-7ktsj namespace: kserve-ci-e2e-test resourceVersion: "19603" uid: af78fa9e-34be-4a42-9242-5feb37d1e364 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-21c95a-predictor-56c84f677d-7ktsj to ip-10-0-133-159.ec2.internal metadata: creationTimestamp: "2026-04-22T19:44:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:44:46Z" name: isvc-init-fail-21c95a-predictor-56c84f677d-7ktsj.18a8c564189730d1 namespace: kserve-ci-e2e-test resourceVersion: "19607" uid: 396042dd-fbe8-497e-a0c3-a7702d6552df reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8fbbbc4c-fn4j6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:47Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-21c95a-predictor-56c84f677d-7ktsj namespace: kserve-ci-e2e-test resourceVersion: "19613" uid: af78fa9e-34be-4a42-9242-5feb37d1e364 kind: Event lastTimestamp: "2026-04-22T19:44:47Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:44:47Z" name: isvc-init-fail-21c95a-predictor-56c84f677d-7ktsj.18a8c5643236f633 namespace: kserve-ci-e2e-test resourceVersion: "19629" uid: 135c4d83-de1b-42f2-a4f1-d337b9288e9b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-21c95a-predictor-56c84f677d-7ktsj namespace: kserve-ci-e2e-test resourceVersion: "19604" uid: af78fa9e-34be-4a42-9242-5feb37d1e364 kind: Event lastTimestamp: "2026-04-22T19:44:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bcc18d6872a741f58ad083af8b7b38f37e5e040b083e0c0f5222178cff3c7b39" already present on machine metadata: creationTimestamp: "2026-04-22T19:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:52Z" name: isvc-init-fail-21c95a-predictor-56c84f677d-7ktsj.18a8c56433a4aca6 namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: 12792e13-5cc4-4554-99e5-b6fd36d6d2c6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-21c95a-predictor-56c84f677d-7ktsj namespace: kserve-ci-e2e-test resourceVersion: "19604" uid: af78fa9e-34be-4a42-9242-5feb37d1e364 kind: Event lastTimestamp: "2026-04-22T19:44:53Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T19:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:53Z" name: isvc-init-fail-21c95a-predictor-56c84f677d-7ktsj.18a8c56436dd9fde namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: 16c95160-8408-4df1-9a26-3f4e8d720646 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-21c95a-predictor-56c84f677d-7ktsj namespace: kserve-ci-e2e-test resourceVersion: "19604" uid: af78fa9e-34be-4a42-9242-5feb37d1e364 kind: Event lastTimestamp: "2026-04-22T19:44:53Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T19:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:53Z" name: isvc-init-fail-21c95a-predictor-56c84f677d-7ktsj.18a8c5643755bc15 namespace: kserve-ci-e2e-test resourceVersion: "19694" uid: fc7fc9b2-aa74-4279-961c-e70672fdd3e0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-21c95a-predictor-56c84f677d-7ktsj namespace: kserve-ci-e2e-test resourceVersion: "19604" uid: af78fa9e-34be-4a42-9242-5feb37d1e364 kind: Event lastTimestamp: "2026-04-22T19:44:56Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-04-22T19:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:56Z" name: isvc-init-fail-21c95a-predictor-56c84f677d-7ktsj.18a8c5666d63d60e namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: 233bc4df-0cc0-4ed9-8c86-9967d18dead6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-21c95a-predictor-56c84f677d namespace: kserve-ci-e2e-test resourceVersion: "19594" uid: 3c5cf882-5aa0-4d3a-8820-e8c978e4ed07 kind: Event lastTimestamp: "2026-04-22T19:44:46Z" message: 'Created pod: isvc-init-fail-21c95a-predictor-56c84f677d-7ktsj' metadata: creationTimestamp: "2026-04-22T19:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:46Z" name: isvc-init-fail-21c95a-predictor-56c84f677d.18a8c5641836fe37 namespace: kserve-ci-e2e-test resourceVersion: "19605" uid: 0f29b680-2308-4395-9c29-0a03bec34cad reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-21c95a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19592" uid: 85d6ec99-4a1e-481b-85b1-eff2472837a4 kind: Event lastTimestamp: "2026-04-22T19:44:46Z" message: Scaled up replica set isvc-init-fail-21c95a-predictor-56c84f677d from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:46Z" name: isvc-init-fail-21c95a-predictor.18a8c56415118ba9 namespace: kserve-ci-e2e-test resourceVersion: "19597" uid: f7f2c266-785a-4c73-8922-b787162ec7d4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-22T19:44:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-21c95a namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: 0ee7cc99-360c-4743-a86f-3ba3b3a1d026 kind: Event lastTimestamp: "2026-04-22T19:44:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:53Z" name: isvc-init-fail-21c95a.18a8c56414e741d2 namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: d7047b31-63d5-415c-bcb6-19ad58f5b02f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:44:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-21c95a namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: 0ee7cc99-360c-4743-a86f-3ba3b3a1d026 kind: Event lastTimestamp: "2026-04-22T19:44:46Z" message: 'Failed to update status for InferenceService "isvc-init-fail-21c95a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-21c95a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:46Z" name: isvc-init-fail-21c95a.18a8c56415fb4650 namespace: kserve-ci-e2e-test resourceVersion: "19615" uid: 422b56ca-7f3a-44fd-bbfc-08f0ad137722 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:44:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-21c95a namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: 0ee7cc99-360c-4743-a86f-3ba3b3a1d026 kind: Event lastTimestamp: "2026-04-22T19:44:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-21c95a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:46Z" name: isvc-init-fail-21c95a.18a8c56415fb8fb9 namespace: kserve-ci-e2e-test resourceVersion: "19617" uid: c022e49b-3931-48f1-a864-58fcafc13927 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:39:14.863012Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns namespace: kserve-ci-e2e-test resourceVersion: "16876" uid: 2bf0aa43-6bf9-49fc-9734-167af6357e21 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-38710-predictor-655997d55d-tm5ns to ip-10-0-133-159.ec2.internal metadata: creationTimestamp: "2026-04-22T19:39:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:39:14Z" name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns.18a8c516d5b4f3d3 namespace: kserve-ci-e2e-test resourceVersion: "16880" uid: dea27119-990e-498d-af9a-86a52e8c9778 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8fbbbc4c-fn4j6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:15Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns namespace: kserve-ci-e2e-test resourceVersion: "16884" uid: 2bf0aa43-6bf9-49fc-9734-167af6357e21 kind: Event lastTimestamp: "2026-04-22T19:39:15Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:39:15Z" name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns.18a8c516ef73ed79 namespace: kserve-ci-e2e-test resourceVersion: "16897" uid: 1a99cbd6-098a-43a2-89c5-3e2ab94ab5f8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: 2bf0aa43-6bf9-49fc-9734-167af6357e21 kind: Event lastTimestamp: "2026-04-22T19:39:15Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bcc18d6872a741f58ad083af8b7b38f37e5e040b083e0c0f5222178cff3c7b39" already present on machine metadata: creationTimestamp: "2026-04-22T19:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:39:15Z" name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns.18a8c516f0ea11d9 namespace: kserve-ci-e2e-test resourceVersion: "16900" uid: 3bc99a55-195d-4206-bf55-3e0866269a3a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: 2bf0aa43-6bf9-49fc-9734-167af6357e21 kind: Event lastTimestamp: "2026-04-22T19:39:15Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T19:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:39:15Z" name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns.18a8c516f45c6d75 namespace: kserve-ci-e2e-test resourceVersion: "16901" uid: c4ba5fb0-f024-4789-98b1-c27d8b2c71a4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: 2bf0aa43-6bf9-49fc-9734-167af6357e21 kind: Event lastTimestamp: "2026-04-22T19:39:15Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T19:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:39:15Z" name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns.18a8c516f4d61a25 namespace: kserve-ci-e2e-test resourceVersion: "16902" uid: 23549888-e414-46a4-86d7-405a03e722ab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: 2bf0aa43-6bf9-49fc-9734-167af6357e21 kind: Event lastTimestamp: "2026-04-22T19:39:19Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:39:19Z" name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns.18a8c5180101ab27 namespace: kserve-ci-e2e-test resourceVersion: "16944" uid: 252188a6-e834-44b3-a5d0-495d933cdb75 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: 2bf0aa43-6bf9-49fc-9734-167af6357e21 kind: Event lastTimestamp: "2026-04-22T19:39:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:39:19Z" name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns.18a8c5180426210c namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: f7c72787-d38e-44fe-ab0c-4cbdb364faa0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: 2bf0aa43-6bf9-49fc-9734-167af6357e21 kind: Event lastTimestamp: "2026-04-22T19:39:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:39:19Z" name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns.18a8c51804a11d7f namespace: kserve-ci-e2e-test resourceVersion: "16949" uid: 6e9f9558-1e1c-47c8-a147-a02c57c2bd8d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: 2bf0aa43-6bf9-49fc-9734-167af6357e21 kind: Event lastTimestamp: "2026-04-22T19:39:19Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:b5c03dc752560187de2d60032e7571fd713c4f5c264bc5211edbec74590c4692" already present on machine metadata: creationTimestamp: "2026-04-22T19:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:39:19Z" name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns.18a8c51804aefe0d namespace: kserve-ci-e2e-test resourceVersion: "16950" uid: 8c43e0ef-a624-4efa-a3d3-1e2f3f00c4b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: 2bf0aa43-6bf9-49fc-9734-167af6357e21 kind: Event lastTimestamp: "2026-04-22T19:39:19Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-22T19:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:39:19Z" name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns.18a8c51807b3dde6 namespace: kserve-ci-e2e-test resourceVersion: "16953" uid: 742edba2-0b86-4968-b490-3339effea8d4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: 2bf0aa43-6bf9-49fc-9734-167af6357e21 kind: Event lastTimestamp: "2026-04-22T19:39:20Z" message: Started container agent metadata: creationTimestamp: "2026-04-22T19:39:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:39:20Z" name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns.18a8c518082641fe namespace: kserve-ci-e2e-test resourceVersion: "16954" uid: 606d2f4c-ab91-495f-b084-df590ed5e90e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-22T19:39:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: 2bf0aa43-6bf9-49fc-9734-167af6357e21 kind: Event lastTimestamp: "2026-04-22T19:41:01Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:39:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:41:01Z" name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns.18a8c5183cec1442 namespace: kserve-ci-e2e-test resourceVersion: "17796" uid: 368707c5-5650-4c69-a73a-3a163bf9390f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-22T19:39:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: 2bf0aa43-6bf9-49fc-9734-167af6357e21 kind: Event lastTimestamp: "2026-04-22T19:41:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:39:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:41:01Z" name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns.18a8c5183cf7e78f namespace: kserve-ci-e2e-test resourceVersion: "17797" uid: 8cdc9e39-c52a-484f-a552-15ff541ab259 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:40:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: 2bf0aa43-6bf9-49fc-9734-167af6357e21 kind: Event lastTimestamp: "2026-04-22T19:40:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:40:40Z" name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns.18a8c52aadbdd995 namespace: kserve-ci-e2e-test resourceVersion: "17576" uid: 41a15649-5d57-4712-8d29-d18677359ba1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:40:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: 2bf0aa43-6bf9-49fc-9734-167af6357e21 kind: Event lastTimestamp: "2026-04-22T19:40:40Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-22T19:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:40:40Z" name: isvc-logger-raw-38710-predictor-655997d55d-tm5ns.18a8c52aadc0b033 namespace: kserve-ci-e2e-test resourceVersion: "17578" uid: e351b714-85e6-4315-9487-6ea1ca066487 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-38710-predictor-655997d55d namespace: kserve-ci-e2e-test resourceVersion: "16864" uid: 9b4b9146-821c-4a2f-b949-62e9f9db3169 kind: Event lastTimestamp: "2026-04-22T19:39:14Z" message: 'Created pod: isvc-logger-raw-38710-predictor-655997d55d-tm5ns' metadata: creationTimestamp: "2026-04-22T19:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:39:14Z" name: isvc-logger-raw-38710-predictor-655997d55d.18a8c516d528580e namespace: kserve-ci-e2e-test resourceVersion: "16877" uid: f60b7f4f-70fd-45cc-8f68-f4fb916ab73d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-38710-predictor namespace: kserve-ci-e2e-test resourceVersion: "16862" uid: b13d0470-7c9e-449a-96cd-9f3c1a133748 kind: Event lastTimestamp: "2026-04-22T19:39:14Z" message: Scaled up replica set isvc-logger-raw-38710-predictor-655997d55d from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:39:14Z" name: isvc-logger-raw-38710-predictor.18a8c516d1fe6381 namespace: kserve-ci-e2e-test resourceVersion: "16867" uid: 0c1ed501-513b-4d47-83b2-6314ecda0893 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:39:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-38710-predictor namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: 74a9d912-8c1b-4aa4-b6f5-a20050c5ac64 kind: Event lastTimestamp: "2026-04-22T19:39:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:39:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:39:44Z" name: isvc-logger-raw-38710-predictor.18a8c51a510271e7 namespace: kserve-ci-e2e-test resourceVersion: "17144" uid: ad8ccab3-9261-40fc-ab57-e5fcd696532e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:39:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-38710-predictor namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: 74a9d912-8c1b-4aa4-b6f5-a20050c5ac64 kind: Event lastTimestamp: "2026-04-22T19:39:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:39:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:39:44Z" name: isvc-logger-raw-38710-predictor.18a8c51a5161e58f namespace: kserve-ci-e2e-test resourceVersion: "17145" uid: 3c8fb000-468a-4ab7-a521-21673f1587b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:39:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-38710-predictor namespace: kserve-ci-e2e-test resourceVersion: "17030" uid: 74a9d912-8c1b-4aa4-b6f5-a20050c5ac64 kind: Event lastTimestamp: "2026-04-22T19:40:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:40:29Z" name: isvc-logger-raw-38710-predictor.18a8c5214f67a9f6 namespace: kserve-ci-e2e-test resourceVersion: "17460" uid: b2ade0eb-306c-40f5-adb6-9add1f71cb81 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:39:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-38710-predictor namespace: kserve-ci-e2e-test resourceVersion: "17030" uid: 74a9d912-8c1b-4aa4-b6f5-a20050c5ac64 kind: Event lastTimestamp: "2026-04-22T19:40:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:40:29Z" name: isvc-logger-raw-38710-predictor.18a8c5214fc90b94 namespace: kserve-ci-e2e-test resourceVersion: "17461" uid: 1084869f-4cf0-4dab-aa41-ae3468a12bfc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-22T19:39:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-38710 namespace: kserve-ci-e2e-test resourceVersion: "16853" uid: e49ddf1c-3af2-45dd-8096-6057fd4ddf02 kind: Event lastTimestamp: "2026-04-22T19:40:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:40:32Z" name: isvc-logger-raw-38710.18a8c516d1dc1f8a namespace: kserve-ci-e2e-test resourceVersion: "17486" uid: a9cd336b-1d03-4c93-86ac-18035c6bef15 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:39:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-38710 namespace: kserve-ci-e2e-test resourceVersion: "16853" uid: e49ddf1c-3af2-45dd-8096-6057fd4ddf02 kind: Event lastTimestamp: "2026-04-22T19:39:14Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-38710": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-38710": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:39:14Z" name: isvc-logger-raw-38710.18a8c516d276e693 namespace: kserve-ci-e2e-test resourceVersion: "16886" uid: d8d811ea-10b5-4636-9bf1-63b984ef6351 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-38710 namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: e49ddf1c-3af2-45dd-8096-6057fd4ddf02 kind: Event lastTimestamp: "2026-04-22T19:39:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-38710": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:39:14Z" name: isvc-logger-raw-38710.18a8c516d78caaec namespace: kserve-ci-e2e-test resourceVersion: "16887" uid: 94fbd0a1-e0be-4020-9f5e-e47687c59fc9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:40:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-38710 namespace: kserve-ci-e2e-test resourceVersion: "17484" uid: e49ddf1c-3af2-45dd-8096-6057fd4ddf02 kind: Event lastTimestamp: "2026-04-22T19:40:31Z" message: InferenceService [isvc-logger-raw-38710] is Ready metadata: creationTimestamp: "2026-04-22T19:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:40:31Z" name: isvc-logger-raw-38710.18a8c528ca992297 namespace: kserve-ci-e2e-test resourceVersion: "17485" uid: 64fde244-7cd0-4879-a206-d55e79257a6f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:43:10.387061Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-cc83d8-predictor-858d4476f-qxlcv namespace: kserve-ci-e2e-test resourceVersion: "18754" uid: 0726536f-15ae-435f-b928-5d765df077ce kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-cc83d8-predictor-858d4476f-qxlcv to ip-10-0-133-159.ec2.internal metadata: creationTimestamp: "2026-04-22T19:43:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:43:10Z" name: isvc-primary-cc83d8-predictor-858d4476f-qxlcv.18a8c54dac08c34a namespace: kserve-ci-e2e-test resourceVersion: "18759" uid: 89c0bcc9-23db-4a72-8bc9-27c9478afe89 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8fbbbc4c-fn4j6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-cc83d8-predictor-858d4476f-qxlcv namespace: kserve-ci-e2e-test resourceVersion: "18763" uid: 0726536f-15ae-435f-b928-5d765df077ce kind: Event lastTimestamp: "2026-04-22T19:43:10Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:43:10Z" name: isvc-primary-cc83d8-predictor-858d4476f-qxlcv.18a8c54dc5caf207 namespace: kserve-ci-e2e-test resourceVersion: "18771" uid: a37b5b56-ee8d-4f2e-a063-b704e747be87 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-cc83d8-predictor-858d4476f-qxlcv namespace: kserve-ci-e2e-test resourceVersion: "18756" uid: 0726536f-15ae-435f-b928-5d765df077ce kind: Event lastTimestamp: "2026-04-22T19:43:10Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bcc18d6872a741f58ad083af8b7b38f37e5e040b083e0c0f5222178cff3c7b39" already present on machine metadata: creationTimestamp: "2026-04-22T19:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:10Z" name: isvc-primary-cc83d8-predictor-858d4476f-qxlcv.18a8c54dc73ada15 namespace: kserve-ci-e2e-test resourceVersion: "18773" uid: 5ae5dbc9-7116-4220-9584-b65dcaa21220 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-cc83d8-predictor-858d4476f-qxlcv namespace: kserve-ci-e2e-test resourceVersion: "18756" uid: 0726536f-15ae-435f-b928-5d765df077ce kind: Event lastTimestamp: "2026-04-22T19:43:10Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T19:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:10Z" name: isvc-primary-cc83d8-predictor-858d4476f-qxlcv.18a8c54dca583f3d namespace: kserve-ci-e2e-test resourceVersion: "18775" uid: 8cfe52ff-ccff-4878-aa85-c646b8e4732c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-cc83d8-predictor-858d4476f-qxlcv namespace: kserve-ci-e2e-test resourceVersion: "18756" uid: 0726536f-15ae-435f-b928-5d765df077ce kind: Event lastTimestamp: "2026-04-22T19:43:10Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T19:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:10Z" name: isvc-primary-cc83d8-predictor-858d4476f-qxlcv.18a8c54dcacf7361 namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: cf7604d5-c52a-418b-8261-8da731f67ff2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cc83d8-predictor-858d4476f-qxlcv namespace: kserve-ci-e2e-test resourceVersion: "18756" uid: 0726536f-15ae-435f-b928-5d765df077ce kind: Event lastTimestamp: "2026-04-22T19:43:15Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:15Z" name: isvc-primary-cc83d8-predictor-858d4476f-qxlcv.18a8c54ee6211aad namespace: kserve-ci-e2e-test resourceVersion: "18816" uid: f8c1597d-4d37-436d-ae39-0f6b0995dc1c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cc83d8-predictor-858d4476f-qxlcv namespace: kserve-ci-e2e-test resourceVersion: "18756" uid: 0726536f-15ae-435f-b928-5d765df077ce kind: Event lastTimestamp: "2026-04-22T19:43:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:15Z" name: isvc-primary-cc83d8-predictor-858d4476f-qxlcv.18a8c54ee9124e74 namespace: kserve-ci-e2e-test resourceVersion: "18820" uid: b98c1beb-d24b-430b-b4a7-ed59672cd9bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cc83d8-predictor-858d4476f-qxlcv namespace: kserve-ci-e2e-test resourceVersion: "18756" uid: 0726536f-15ae-435f-b928-5d765df077ce kind: Event lastTimestamp: "2026-04-22T19:43:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:15Z" name: isvc-primary-cc83d8-predictor-858d4476f-qxlcv.18a8c54ee98e5208 namespace: kserve-ci-e2e-test resourceVersion: "18822" uid: dd2f74c0-5f70-4732-ad78-c060379784c4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-22T19:43:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cc83d8-predictor-858d4476f-qxlcv namespace: kserve-ci-e2e-test resourceVersion: "18756" uid: 0726536f-15ae-435f-b928-5d765df077ce kind: Event lastTimestamp: "2026-04-22T19:44:47Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:47Z" name: isvc-primary-cc83d8-predictor-858d4476f-qxlcv.18a8c54f2203f1bb namespace: kserve-ci-e2e-test resourceVersion: "19637" uid: e51d5ffe-a94e-428c-aba9-0545b5fd7242 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cc83d8-predictor-858d4476f-qxlcv namespace: kserve-ci-e2e-test resourceVersion: "18756" uid: 0726536f-15ae-435f-b928-5d765df077ce kind: Event lastTimestamp: "2026-04-22T19:44:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:46Z" name: isvc-primary-cc83d8-predictor-858d4476f-qxlcv.18a8c5640fcc608a namespace: kserve-ci-e2e-test resourceVersion: "19567" uid: 0b9d2fca-0940-4708-8c14-9240b8d27926 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-cc83d8-predictor-858d4476f namespace: kserve-ci-e2e-test resourceVersion: "18744" uid: f94a4537-b7fa-43d7-8086-ac8d6acffb57 kind: Event lastTimestamp: "2026-04-22T19:43:10Z" message: 'Created pod: isvc-primary-cc83d8-predictor-858d4476f-qxlcv' metadata: creationTimestamp: "2026-04-22T19:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:10Z" name: isvc-primary-cc83d8-predictor-858d4476f.18a8c54dab8afcf5 namespace: kserve-ci-e2e-test resourceVersion: "18755" uid: 7b930f02-87ef-4795-b1d4-49245f68efdb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-cc83d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: e02bb2d2-1551-4bf1-b971-95535c6320ae kind: Event lastTimestamp: "2026-04-22T19:43:10Z" message: Scaled up replica set isvc-primary-cc83d8-predictor-858d4476f from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:10Z" name: isvc-primary-cc83d8-predictor.18a8c54da7401bda namespace: kserve-ci-e2e-test resourceVersion: "18747" uid: 46d71626-9063-4cf3-8f6c-0de347e9df07 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:43:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-cc83d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "18743" uid: 55ad3a00-ab0d-4380-a849-2136240ab1dc kind: Event lastTimestamp: "2026-04-22T19:43:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:40Z" name: isvc-primary-cc83d8-predictor.18a8c551262d6c5e namespace: kserve-ci-e2e-test resourceVersion: "19002" uid: 6e3064e0-b674-4fbb-a195-bfb910dde2ff reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:43:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-cc83d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "18743" uid: 55ad3a00-ab0d-4380-a849-2136240ab1dc kind: Event lastTimestamp: "2026-04-22T19:43:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:40Z" name: isvc-primary-cc83d8-predictor.18a8c55126834e14 namespace: kserve-ci-e2e-test resourceVersion: "19003" uid: ac0039ee-c718-4e59-9f62-dd1310d1cb86 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:43:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-cc83d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "18894" uid: 55ad3a00-ab0d-4380-a849-2136240ab1dc kind: Event lastTimestamp: "2026-04-22T19:44:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:43:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:40Z" name: isvc-primary-cc83d8-predictor.18a8c558248e904f namespace: kserve-ci-e2e-test resourceVersion: "19495" uid: 10abd571-c9a9-4d2b-b583-120757d1628a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:43:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-cc83d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "18894" uid: 55ad3a00-ab0d-4380-a849-2136240ab1dc kind: Event lastTimestamp: "2026-04-22T19:44:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:43:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:40Z" name: isvc-primary-cc83d8-predictor.18a8c55824ea889e namespace: kserve-ci-e2e-test resourceVersion: "19496" uid: e89e317b-1ac0-4330-95d5-c2be1d993b2f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-22T19:43:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-cc83d8 namespace: kserve-ci-e2e-test resourceVersion: "18717" uid: 14ca7942-d29e-4115-95a2-11f832ae97c3 kind: Event lastTimestamp: "2026-04-22T19:44:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:27Z" name: isvc-primary-cc83d8.18a8c54da7201b3a namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: f555739e-ddd6-4c02-bb7c-85e7abf0db97 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-cc83d8 namespace: kserve-ci-e2e-test resourceVersion: "18717" uid: 14ca7942-d29e-4115-95a2-11f832ae97c3 kind: Event lastTimestamp: "2026-04-22T19:43:10Z" message: 'Failed to update status for InferenceService "isvc-primary-cc83d8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-cc83d8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:43:10Z" name: isvc-primary-cc83d8.18a8c54da7825816 namespace: kserve-ci-e2e-test resourceVersion: "18749" uid: a31171bf-2c10-4a5b-995d-b1645ac3d693 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-cc83d8 namespace: kserve-ci-e2e-test resourceVersion: "18717" uid: 14ca7942-d29e-4115-95a2-11f832ae97c3 kind: Event lastTimestamp: "2026-04-22T19:43:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-cc83d8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:43:10Z" name: isvc-primary-cc83d8.18a8c54da782b578 namespace: kserve-ci-e2e-test resourceVersion: "18750" uid: 65b04abc-fe66-4ee0-86c5-e8e890e62ae8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-cc83d8 namespace: kserve-ci-e2e-test resourceVersion: "19343" uid: 14ca7942-d29e-4115-95a2-11f832ae97c3 kind: Event lastTimestamp: "2026-04-22T19:44:27Z" message: InferenceService [isvc-primary-cc83d8] is Ready metadata: creationTimestamp: "2026-04-22T19:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:27Z" name: isvc-primary-cc83d8.18a8c55fae349f0b namespace: kserve-ci-e2e-test resourceVersion: "19344" uid: a352b79d-d1f7-4f35-980a-a928aa49aa6e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:33:49.101769Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq namespace: kserve-ci-e2e-test resourceVersion: "13921" uid: 6a215e8e-0981-4f2f-bc93-4abe72b71b8f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq to ip-10-0-133-159.ec2.internal metadata: creationTimestamp: "2026-04-22T19:33:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:33:49Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq.18a8c4cafcd2e658 namespace: kserve-ci-e2e-test resourceVersion: "13927" uid: 862b5d1e-042e-4350-a4e7-717e8a44583b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8fbbbc4c-fn4j6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:49Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq namespace: kserve-ci-e2e-test resourceVersion: "13933" uid: 6a215e8e-0981-4f2f-bc93-4abe72b71b8f kind: Event lastTimestamp: "2026-04-22T19:33:49Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:33:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:33:49Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq.18a8c4cb15ff2e27 namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 059ff2c0-687d-47b2-9151-ca438efdb549 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 6a215e8e-0981-4f2f-bc93-4abe72b71b8f kind: Event lastTimestamp: "2026-04-22T19:33:49Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:bcc18d6872a741f58ad083af8b7b38f37e5e040b083e0c0f5222178cff3c7b39" metadata: creationTimestamp: "2026-04-22T19:33:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:49Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq.18a8c4cb175f09e6 namespace: kserve-ci-e2e-test resourceVersion: "13944" uid: 4f318870-f356-4590-80bf-506d1f1e78ec reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 6a215e8e-0981-4f2f-bc93-4abe72b71b8f kind: Event lastTimestamp: "2026-04-22T19:33:52Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:bcc18d6872a741f58ad083af8b7b38f37e5e040b083e0c0f5222178cff3c7b39" in 3.125s (3.125s including waiting). Image size: 299660181 bytes.' metadata: creationTimestamp: "2026-04-22T19:33:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:52Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq.18a8c4cbd1b11c20 namespace: kserve-ci-e2e-test resourceVersion: "13967" uid: 0b8cbe66-7862-49c4-a1ae-f80fa29716a0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 6a215e8e-0981-4f2f-bc93-4abe72b71b8f kind: Event lastTimestamp: "2026-04-22T19:33:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T19:33:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:52Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq.18a8c4cbd4e413e4 namespace: kserve-ci-e2e-test resourceVersion: "13968" uid: 12c68ba2-2bc3-4a0d-9ece-3c3ffc1d8160 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 6a215e8e-0981-4f2f-bc93-4abe72b71b8f kind: Event lastTimestamp: "2026-04-22T19:33:52Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T19:33:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:52Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq.18a8c4cbd5799f5b namespace: kserve-ci-e2e-test resourceVersion: "13969" uid: 12fa7eee-ee5e-40ac-a99f-14b970549ceb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 6a215e8e-0981-4f2f-bc93-4abe72b71b8f kind: Event lastTimestamp: "2026-04-22T19:33:56Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1256" metadata: creationTimestamp: "2026-04-22T19:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:33:56Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq.18a8c4ccc6186086 namespace: kserve-ci-e2e-test resourceVersion: "14003" uid: 7e93e897-3636-4deb-8e97-0a67294b8af0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 6a215e8e-0981-4f2f-bc93-4abe72b71b8f kind: Event lastTimestamp: "2026-04-22T19:34:09Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1256" in 13.133s (13.133s including waiting). Image size: 1565185957 bytes.' metadata: creationTimestamp: "2026-04-22T19:34:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:09Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq.18a8c4cfd4ec21e7 namespace: kserve-ci-e2e-test resourceVersion: "14098" uid: d9d8966e-8f25-466f-a104-abcbadf547bd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 6a215e8e-0981-4f2f-bc93-4abe72b71b8f kind: Event lastTimestamp: "2026-04-22T19:34:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:34:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:09Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq.18a8c4cfd8238e5f namespace: kserve-ci-e2e-test resourceVersion: "14100" uid: adaecfcb-4085-4703-8670-182f2fc1ee6b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 6a215e8e-0981-4f2f-bc93-4abe72b71b8f kind: Event lastTimestamp: "2026-04-22T19:34:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:34:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:09Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq.18a8c4cfd8982aaf namespace: kserve-ci-e2e-test resourceVersion: "14101" uid: f3afbaa6-772e-4c87-b8dc-711383efd7a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 6a215e8e-0981-4f2f-bc93-4abe72b71b8f kind: Event lastTimestamp: "2026-04-22T19:34:09Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:b5c03dc752560187de2d60032e7571fd713c4f5c264bc5211edbec74590c4692" metadata: creationTimestamp: "2026-04-22T19:34:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:09Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq.18a8c4cfd8a43dcf namespace: kserve-ci-e2e-test resourceVersion: "14102" uid: f32ac52b-aa2b-4bf6-a42a-b11a9a7fa0bc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 6a215e8e-0981-4f2f-bc93-4abe72b71b8f kind: Event lastTimestamp: "2026-04-22T19:34:13Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:b5c03dc752560187de2d60032e7571fd713c4f5c264bc5211edbec74590c4692" in 3.329s (3.329s including waiting). Image size: 237896250 bytes.' metadata: creationTimestamp: "2026-04-22T19:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:13Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq.18a8c4d09f1cd413 namespace: kserve-ci-e2e-test resourceVersion: "14124" uid: 5841c74c-4030-4404-8db9-9e06e87bb749 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 6a215e8e-0981-4f2f-bc93-4abe72b71b8f kind: Event lastTimestamp: "2026-04-22T19:34:13Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-22T19:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:13Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq.18a8c4d0a2b21e0e namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: 7375482d-d202-4317-8b06-b2190ae39b3e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 6a215e8e-0981-4f2f-bc93-4abe72b71b8f kind: Event lastTimestamp: "2026-04-22T19:34:13Z" message: Started container agent metadata: creationTimestamp: "2026-04-22T19:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:13Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq.18a8c4d0a3259137 namespace: kserve-ci-e2e-test resourceVersion: "14127" uid: fff59c81-bfa6-4b60-a11c-8f1faf069814 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-22T19:34:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 6a215e8e-0981-4f2f-bc93-4abe72b71b8f kind: Event lastTimestamp: "2026-04-22T19:35:55Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:35:55Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq.18a8c4d0bf533a8b namespace: kserve-ci-e2e-test resourceVersion: "15002" uid: 4d7db577-87b3-468e-810b-e8d250faddb0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-22T19:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 6a215e8e-0981-4f2f-bc93-4abe72b71b8f kind: Event lastTimestamp: "2026-04-22T19:35:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:35:55Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq.18a8c4d0fb350d6c namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 5c6bf608-6fd0-4e04-8aa9-8201e301d5db reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 6a215e8e-0981-4f2f-bc93-4abe72b71b8f kind: Event lastTimestamp: "2026-04-22T19:35:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:35:34Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq.18a8c4e371f97ed3 namespace: kserve-ci-e2e-test resourceVersion: "14743" uid: e3e1e571-592d-46d3-afba-a89b734b03c9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 6a215e8e-0981-4f2f-bc93-4abe72b71b8f kind: Event lastTimestamp: "2026-04-22T19:35:34Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-22T19:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:35:34Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq.18a8c4e371fa9e9a namespace: kserve-ci-e2e-test resourceVersion: "14746" uid: 13ac0172-18d1-41bc-bc52-4b5cf9cf62fe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86 namespace: kserve-ci-e2e-test resourceVersion: "13913" uid: 56282dbe-e82c-4aff-8950-ca8e6b93bca5 kind: Event lastTimestamp: "2026-04-22T19:33:49Z" message: 'Created pod: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86-xj7nq' metadata: creationTimestamp: "2026-04-22T19:33:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:49Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86.18a8c4cafbdd683c namespace: kserve-ci-e2e-test resourceVersion: "13924" uid: 682ad635-a757-4797-a95e-bcae4badb31a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-5fa8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13912" uid: 0b2abb84-7e11-46ce-a010-8ba400981a0e kind: Event lastTimestamp: "2026-04-22T19:33:48Z" message: Scaled up replica set isvc-raw-sklearn-batcher-5fa8b-predictor-6b99545d86 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:33:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:33:48Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor.18a8c4caf6177eaa namespace: kserve-ci-e2e-test resourceVersion: "13915" uid: 21288012-3251-44fc-a6b7-29969716ca1d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:34:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-5fa8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13922" uid: 8e188919-4342-48ef-9d82-a09a39f43d57 kind: Event lastTimestamp: "2026-04-22T19:34:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:34:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:34:19Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor.18a8c4ce7c757de6 namespace: kserve-ci-e2e-test resourceVersion: "14173" uid: 0443124d-edcc-4fd9-bf25-67f25acd2292 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:34:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-5fa8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13922" uid: 8e188919-4342-48ef-9d82-a09a39f43d57 kind: Event lastTimestamp: "2026-04-22T19:34:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:34:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:34:19Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor.18a8c4ce7cf30c7c namespace: kserve-ci-e2e-test resourceVersion: "14174" uid: 963fe25e-bd59-4921-9452-c1de236ebaa4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:34:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-5fa8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "14059" uid: 8e188919-4342-48ef-9d82-a09a39f43d57 kind: Event lastTimestamp: "2026-04-22T19:35:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:34:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:35:19Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor.18a8c4d57b1e1fcd namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 89873d40-5637-488b-b23d-60d048893d50 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:34:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-5fa8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "14059" uid: 8e188919-4342-48ef-9d82-a09a39f43d57 kind: Event lastTimestamp: "2026-04-22T19:35:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:34:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:35:19Z" name: isvc-raw-sklearn-batcher-5fa8b-predictor.18a8c4d57b933576 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 83d926ca-3e4e-4630-9cfa-3abdce350318 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-22T19:33:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-5fa8b namespace: kserve-ci-e2e-test resourceVersion: "13900" uid: 899061fd-db2a-4e7e-8020-94e82f09b4a8 kind: Event lastTimestamp: "2026-04-22T19:35:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:33:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:35:25Z" name: isvc-raw-sklearn-batcher-5fa8b.18a8c4cafcaa0548 namespace: kserve-ci-e2e-test resourceVersion: "14657" uid: a25e5c0d-f66b-4c60-8181-d2960d7d53b7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-5fa8b namespace: kserve-ci-e2e-test resourceVersion: "13900" uid: 899061fd-db2a-4e7e-8020-94e82f09b4a8 kind: Event lastTimestamp: "2026-04-22T19:33:49Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-5fa8b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-5fa8b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:33:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:33:49Z" name: isvc-raw-sklearn-batcher-5fa8b.18a8c4cafd4da68e namespace: kserve-ci-e2e-test resourceVersion: "13930" uid: 398b99c4-4f9b-458a-8348-2c430021f5e8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:33:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-5fa8b namespace: kserve-ci-e2e-test resourceVersion: "13900" uid: 899061fd-db2a-4e7e-8020-94e82f09b4a8 kind: Event lastTimestamp: "2026-04-22T19:33:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-5fa8b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:33:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:33:49Z" name: isvc-raw-sklearn-batcher-5fa8b.18a8c4cafd4dfb89 namespace: kserve-ci-e2e-test resourceVersion: "13932" uid: 70b21835-15e3-4c47-b7cc-cca02dfddc01 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-5fa8b namespace: kserve-ci-e2e-test resourceVersion: "14654" uid: 899061fd-db2a-4e7e-8020-94e82f09b4a8 kind: Event lastTimestamp: "2026-04-22T19:35:25Z" message: InferenceService [isvc-raw-sklearn-batcher-5fa8b] is Ready metadata: creationTimestamp: "2026-04-22T19:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:35:25Z" name: isvc-raw-sklearn-batcher-5fa8b.18a8c4e18889661a namespace: kserve-ci-e2e-test resourceVersion: "14655" uid: 72dd4fd3-3989-4974-98e3-d71482263ba1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:44:30.514040Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-cc83d8-predictor-75df94df6d-2dcrg namespace: kserve-ci-e2e-test resourceVersion: "19390" uid: 065d7847-cf9d-443f-b908-51d3f0f6786a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-cc83d8-predictor-75df94df6d-2dcrg to ip-10-0-133-159.ec2.internal metadata: creationTimestamp: "2026-04-22T19:44:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:44:30Z" name: isvc-secondary-cc83d8-predictor-75df94df6d-2dcrg.18a8c56053f96a55 namespace: kserve-ci-e2e-test resourceVersion: "19395" uid: 59660dae-72f5-49c5-80ec-ee38c1c69646 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8fbbbc4c-fn4j6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:30Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-cc83d8-predictor-75df94df6d-2dcrg namespace: kserve-ci-e2e-test resourceVersion: "19399" uid: 065d7847-cf9d-443f-b908-51d3f0f6786a kind: Event lastTimestamp: "2026-04-22T19:44:30Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:44:30Z" name: isvc-secondary-cc83d8-predictor-75df94df6d-2dcrg.18a8c5606d957a3a namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: 0d05d2a7-631c-4136-b5d3-13fa60093e49 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:44:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-cc83d8-predictor-75df94df6d-2dcrg namespace: kserve-ci-e2e-test resourceVersion: "19392" uid: 065d7847-cf9d-443f-b908-51d3f0f6786a kind: Event lastTimestamp: "2026-04-22T19:44:34Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bcc18d6872a741f58ad083af8b7b38f37e5e040b083e0c0f5222178cff3c7b39" already present on machine metadata: creationTimestamp: "2026-04-22T19:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:34Z" name: isvc-secondary-cc83d8-predictor-75df94df6d-2dcrg.18a8c5606f0adf65 namespace: kserve-ci-e2e-test resourceVersion: "19445" uid: 5de9f2bb-fd90-4d6a-a2d3-8145e4e62075 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:44:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-cc83d8-predictor-75df94df6d-2dcrg namespace: kserve-ci-e2e-test resourceVersion: "19392" uid: 065d7847-cf9d-443f-b908-51d3f0f6786a kind: Event lastTimestamp: "2026-04-22T19:44:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T19:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:34Z" name: isvc-secondary-cc83d8-predictor-75df94df6d-2dcrg.18a8c56072430a84 namespace: kserve-ci-e2e-test resourceVersion: "19449" uid: a159c832-1ae7-44e9-ae21-424644664b7d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:44:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-cc83d8-predictor-75df94df6d-2dcrg namespace: kserve-ci-e2e-test resourceVersion: "19392" uid: 065d7847-cf9d-443f-b908-51d3f0f6786a kind: Event lastTimestamp: "2026-04-22T19:44:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T19:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:34Z" name: isvc-secondary-cc83d8-predictor-75df94df6d-2dcrg.18a8c56072bb6091 namespace: kserve-ci-e2e-test resourceVersion: "19450" uid: 7c56d0b2-161a-4ec8-b74d-dbadabae0c59 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-cc83d8-predictor-75df94df6d-2dcrg namespace: kserve-ci-e2e-test resourceVersion: "19392" uid: 065d7847-cf9d-443f-b908-51d3f0f6786a kind: Event lastTimestamp: "2026-04-22T19:44:39Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-cc83d8-predictor-75df94df6d-2dcrg_kserve-ci-e2e-test(065d7847-cf9d-443f-b908-51d3f0f6786a) metadata: creationTimestamp: "2026-04-22T19:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:39Z" name: isvc-secondary-cc83d8-predictor-75df94df6d-2dcrg.18a8c562860e295f namespace: kserve-ci-e2e-test resourceVersion: "19488" uid: e2fe4b8c-e512-401b-b144-93605d528f13 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-cc83d8-predictor-75df94df6d namespace: kserve-ci-e2e-test resourceVersion: "19380" uid: 03751f67-7814-487e-bdba-5b81f878e588 kind: Event lastTimestamp: "2026-04-22T19:44:30Z" message: 'Created pod: isvc-secondary-cc83d8-predictor-75df94df6d-2dcrg' metadata: creationTimestamp: "2026-04-22T19:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:30Z" name: isvc-secondary-cc83d8-predictor-75df94df6d.18a8c560538ba38b namespace: kserve-ci-e2e-test resourceVersion: "19391" uid: a8e829af-b5c2-4e16-b639-9a0095c95070 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-cc83d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19378" uid: c4945113-593c-478e-9e96-cfab2de2f231 kind: Event lastTimestamp: "2026-04-22T19:44:30Z" message: Scaled up replica set isvc-secondary-cc83d8-predictor-75df94df6d from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:30Z" name: isvc-secondary-cc83d8-predictor.18a8c560509ec64e namespace: kserve-ci-e2e-test resourceVersion: "19382" uid: 011e1bf6-02ac-46ee-9cd0-887365856218 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-cc83d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19379" uid: a2dbaa76-d87b-47e5-8048-54e6180107ef kind: Event lastTimestamp: "2026-04-22T19:44:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:44:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:45Z" name: isvc-secondary-cc83d8-predictor.18a8c563cf7a2a0d namespace: kserve-ci-e2e-test resourceVersion: "19532" uid: 5472b155-7256-4fa4-a3a4-4165a4250705 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-cc83d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19379" uid: a2dbaa76-d87b-47e5-8048-54e6180107ef kind: Event lastTimestamp: "2026-04-22T19:44:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:44:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:45Z" name: isvc-secondary-cc83d8-predictor.18a8c563d0afda3d namespace: kserve-ci-e2e-test resourceVersion: "19534" uid: 039d6482-1a91-4f82-a96a-599ca7de5705 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-22T19:44:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-cc83d8 namespace: kserve-ci-e2e-test resourceVersion: "19369" uid: 2b664093-8aa6-4ec1-a51c-9479a9b4e2d9 kind: Event lastTimestamp: "2026-04-22T19:44:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:41Z" name: isvc-secondary-cc83d8.18a8c560507aeafd namespace: kserve-ci-e2e-test resourceVersion: "19504" uid: b30840cd-3548-4cd3-8139-b068d24c25c4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-cc83d8 namespace: kserve-ci-e2e-test resourceVersion: "19369" uid: 2b664093-8aa6-4ec1-a51c-9479a9b4e2d9 kind: Event lastTimestamp: "2026-04-22T19:44:30Z" message: 'Failed to update status for InferenceService "isvc-secondary-cc83d8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-cc83d8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:30Z" name: isvc-secondary-cc83d8.18a8c56050d518ba namespace: kserve-ci-e2e-test resourceVersion: "19384" uid: 2d9da5f5-8555-423f-a5dd-9834f3aa6227 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-cc83d8 namespace: kserve-ci-e2e-test resourceVersion: "19369" uid: 2b664093-8aa6-4ec1-a51c-9479a9b4e2d9 kind: Event lastTimestamp: "2026-04-22T19:44:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-cc83d8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:30Z" name: isvc-secondary-cc83d8.18a8c56050d573cd namespace: kserve-ci-e2e-test resourceVersion: "19385" uid: 4ca33ab4-4f39-4f42-9912-9978eda6e2d7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:35:34.283574Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: 8ebb5400-0432-4003-89da-f25fa6c3abe1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf to ip-10-0-133-159.ec2.internal metadata: creationTimestamp: "2026-04-22T19:35:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:35:34Z" name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf.18a8c4e37a25e3ee namespace: kserve-ci-e2e-test resourceVersion: "14775" uid: 4e2e9103-9330-4a0c-83ac-9d1b5294c7ba reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8fbbbc4c-fn4j6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:34Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: 8ebb5400-0432-4003-89da-f25fa6c3abe1 kind: Event lastTimestamp: "2026-04-22T19:35:34Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:35:34Z" name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf.18a8c4e392e2e87b namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: 07353fee-5b1c-487a-be3a-e32166266950 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf namespace: kserve-ci-e2e-test resourceVersion: "14767" uid: 8ebb5400-0432-4003-89da-f25fa6c3abe1 kind: Event lastTimestamp: "2026-04-22T19:35:34Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bcc18d6872a741f58ad083af8b7b38f37e5e040b083e0c0f5222178cff3c7b39" already present on machine metadata: creationTimestamp: "2026-04-22T19:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:35:34Z" name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf.18a8c4e394912535 namespace: kserve-ci-e2e-test resourceVersion: "14807" uid: ee8185e0-773c-4b93-80e0-676d120b06de reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf namespace: kserve-ci-e2e-test resourceVersion: "14767" uid: 8ebb5400-0432-4003-89da-f25fa6c3abe1 kind: Event lastTimestamp: "2026-04-22T19:35:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T19:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:35:34Z" name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf.18a8c4e397dff6bc namespace: kserve-ci-e2e-test resourceVersion: "14808" uid: ae32f2a4-f861-4313-a18e-2308bb355aed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf namespace: kserve-ci-e2e-test resourceVersion: "14767" uid: 8ebb5400-0432-4003-89da-f25fa6c3abe1 kind: Event lastTimestamp: "2026-04-22T19:35:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T19:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:35:34Z" name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf.18a8c4e398579c59 namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: c1cc46ac-3487-46be-b443-7b4d89641331 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf namespace: kserve-ci-e2e-test resourceVersion: "14767" uid: 8ebb5400-0432-4003-89da-f25fa6c3abe1 kind: Event lastTimestamp: "2026-04-22T19:35:39Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:35:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:35:39Z" name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf.18a8c4e49a0778aa namespace: kserve-ci-e2e-test resourceVersion: "14863" uid: 05908f24-b000-4134-9e17-791eab0d5a32 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf namespace: kserve-ci-e2e-test resourceVersion: "14767" uid: 8ebb5400-0432-4003-89da-f25fa6c3abe1 kind: Event lastTimestamp: "2026-04-22T19:35:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:35:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:35:39Z" name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf.18a8c4e49d86854a namespace: kserve-ci-e2e-test resourceVersion: "14869" uid: 7724fa1e-060a-4b67-838b-8c22a342e1fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf namespace: kserve-ci-e2e-test resourceVersion: "14767" uid: 8ebb5400-0432-4003-89da-f25fa6c3abe1 kind: Event lastTimestamp: "2026-04-22T19:35:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:35:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:35:39Z" name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf.18a8c4e49df5ee42 namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: ec39c86f-6860-43a3-b989-51b5ab049999 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-22T19:35:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf namespace: kserve-ci-e2e-test resourceVersion: "14767" uid: 8ebb5400-0432-4003-89da-f25fa6c3abe1 kind: Event lastTimestamp: "2026-04-22T19:37:18Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:35:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:18Z" name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf.18a8c4e4d6317513 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: b9935ac5-6424-489d-ac91-e62764577bfd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf namespace: kserve-ci-e2e-test resourceVersion: "14767" uid: 8ebb5400-0432-4003-89da-f25fa6c3abe1 kind: Event lastTimestamp: "2026-04-22T19:37:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:14Z" name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf.18a8c4face3698c7 namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: 6fd9acd5-0922-436f-b090-dfab90f36bd0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 2fa0b292-39c5-44b6-8f11-b1efd6cb5f30 kind: Event lastTimestamp: "2026-04-22T19:35:34Z" message: 'Created pod: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb-722tf' metadata: creationTimestamp: "2026-04-22T19:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:35:34Z" name: isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb.18a8c4e37856797d namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: 9dca4109-68c0-45c0-a303-da87a3a70cf6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-8e08c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 8b45576f-7e72-4d4b-b436-c91d9791b3f7 kind: Event lastTimestamp: "2026-04-22T19:35:34Z" message: Scaled up replica set isvc-sklearn-graph-raw-8e08c-predictor-5985d486cb from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:35:34Z" name: isvc-sklearn-graph-raw-8e08c-predictor.18a8c4e371ecaaba namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: 7e2d4551-1fd5-474c-acbb-9a5af6354ec4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-8e08c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14741" uid: eb2b3e68-da68-4246-b429-aaebb0e3a624 kind: Event lastTimestamp: "2026-04-22T19:35:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:35:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:35:49Z" name: isvc-sklearn-graph-raw-8e08c-predictor.18a8c4e6f09de2a0 namespace: kserve-ci-e2e-test resourceVersion: "14950" uid: 2f5c433d-8708-497e-a7ff-6831d3916aee reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-8e08c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14741" uid: eb2b3e68-da68-4246-b429-aaebb0e3a624 kind: Event lastTimestamp: "2026-04-22T19:35:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:35:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:35:49Z" name: isvc-sklearn-graph-raw-8e08c-predictor.18a8c4e6f103965d namespace: kserve-ci-e2e-test resourceVersion: "14952" uid: fad169c7-c80e-4980-b46f-2bed0dc8aa1e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:36:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-8e08c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: eb2b3e68-da68-4246-b429-aaebb0e3a624 kind: Event lastTimestamp: "2026-04-22T19:37:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:36:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:37:04Z" name: isvc-sklearn-graph-raw-8e08c-predictor.18a8c4ea703d7a4d namespace: kserve-ci-e2e-test resourceVersion: "15541" uid: 0b37970c-cb4e-4fbf-942e-d9c6c347681d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:36:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-8e08c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: eb2b3e68-da68-4246-b429-aaebb0e3a624 kind: Event lastTimestamp: "2026-04-22T19:37:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:36:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:37:04Z" name: isvc-sklearn-graph-raw-8e08c-predictor.18a8c4ea70a32069 namespace: kserve-ci-e2e-test resourceVersion: "15546" uid: 1a61b463-8164-4605-9677-2bd8a4c297fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T19:35:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-8e08c namespace: kserve-ci-e2e-test resourceVersion: "14715" uid: 1f37ceb0-3e02-439c-a4d3-06c4679fd026 kind: Event lastTimestamp: "2026-04-22T19:36:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:36:48Z" name: isvc-sklearn-graph-raw-8e08c.18a8c4e371f8b8da namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 7c85c6f4-4714-4d16-bcb4-f08923d6d01d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-8e08c namespace: kserve-ci-e2e-test resourceVersion: "14715" uid: 1f37ceb0-3e02-439c-a4d3-06c4679fd026 kind: Event lastTimestamp: "2026-04-22T19:35:34Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-8e08c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-8e08c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:35:34Z" name: isvc-sklearn-graph-raw-8e08c.18a8c4e3728b5e36 namespace: kserve-ci-e2e-test resourceVersion: "14747" uid: e679b3f0-e637-434f-8921-b1e35d07f389 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:36:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-8e08c namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: 1f37ceb0-3e02-439c-a4d3-06c4679fd026 kind: Event lastTimestamp: "2026-04-22T19:36:48Z" message: InferenceService [isvc-sklearn-graph-raw-8e08c] is Ready metadata: creationTimestamp: "2026-04-22T19:36:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:36:48Z" name: isvc-sklearn-graph-raw-8e08c.18a8c4f4d807dd66 namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 6aeb54e6-e05e-4a67-bcd1-4c8eecd2897b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:37:14.534934Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc namespace: kserve-ci-e2e-test resourceVersion: "15697" uid: f536e031-82ee-46e3-8ca7-d53ad0285015 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc to ip-10-0-133-159.ec2.internal metadata: creationTimestamp: "2026-04-22T19:37:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:37:14Z" name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc.18a8c4fad1983926 namespace: kserve-ci-e2e-test resourceVersion: "15704" uid: a848d5d1-3337-4485-a8e9-1cac63a17d9c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8fbbbc4c-fn4j6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:14Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc namespace: kserve-ci-e2e-test resourceVersion: "15714" uid: f536e031-82ee-46e3-8ca7-d53ad0285015 kind: Event lastTimestamp: "2026-04-22T19:37:14Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:37:14Z" name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc.18a8c4faeb779b54 namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: 91b6adf4-5f22-4823-9b09-14630c348c74 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: f536e031-82ee-46e3-8ca7-d53ad0285015 kind: Event lastTimestamp: "2026-04-22T19:37:14Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bcc18d6872a741f58ad083af8b7b38f37e5e040b083e0c0f5222178cff3c7b39" already present on machine metadata: creationTimestamp: "2026-04-22T19:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:14Z" name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc.18a8c4faecc68b37 namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: 81b20b56-d43d-48e8-947e-71c21674cc95 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: f536e031-82ee-46e3-8ca7-d53ad0285015 kind: Event lastTimestamp: "2026-04-22T19:37:15Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T19:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:15Z" name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc.18a8c4faf0d27efc namespace: kserve-ci-e2e-test resourceVersion: "15765" uid: 852bb0f4-a7e4-48b9-aa71-bc65b799107d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: f536e031-82ee-46e3-8ca7-d53ad0285015 kind: Event lastTimestamp: "2026-04-22T19:37:15Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T19:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:15Z" name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc.18a8c4faf13837f5 namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 741c44bf-a13c-4e74-ab43-c15a6703ca8d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: f536e031-82ee-46e3-8ca7-d53ad0285015 kind: Event lastTimestamp: "2026-04-22T19:37:19Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:37:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:19Z" name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc.18a8c4fbf743b571 namespace: kserve-ci-e2e-test resourceVersion: "15816" uid: 08ccf07a-d72d-421b-8911-32593190b94e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: f536e031-82ee-46e3-8ca7-d53ad0285015 kind: Event lastTimestamp: "2026-04-22T19:37:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:37:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:19Z" name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc.18a8c4fbfb5af0c3 namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: d945c3eb-5bae-4d83-a538-4872ee0c4736 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: f536e031-82ee-46e3-8ca7-d53ad0285015 kind: Event lastTimestamp: "2026-04-22T19:37:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:37:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:19Z" name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc.18a8c4fbfbd3a203 namespace: kserve-ci-e2e-test resourceVersion: "15830" uid: ec586661-2272-47c3-8742-f0062e070aa8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-22T19:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: f536e031-82ee-46e3-8ca7-d53ad0285015 kind: Event lastTimestamp: "2026-04-22T19:38:24Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:38:24Z" name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc.18a8c4fc337d703e namespace: kserve-ci-e2e-test resourceVersion: "16339" uid: e02b1873-705b-4137-be1c-7c7bbdfc874c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: f536e031-82ee-46e3-8ca7-d53ad0285015 kind: Event lastTimestamp: "2026-04-22T19:39:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:39:04Z" name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc.18a8c5147d0eefa4 namespace: kserve-ci-e2e-test resourceVersion: "16719" uid: d6440527-a6b2-4b60-89bb-41d55194ea0a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84 namespace: kserve-ci-e2e-test resourceVersion: "15672" uid: b0e5967d-88a8-4e71-afcf-61e2e6e69365 kind: Event lastTimestamp: "2026-04-22T19:37:14Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84-772mc' metadata: creationTimestamp: "2026-04-22T19:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:37:14Z" name: isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84.18a8c4fad11ab634 namespace: kserve-ci-e2e-test resourceVersion: "15700" uid: 78f48d62-09c8-4d85-ad48-fa1b9b2ddee2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-ed925-predictor namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: c4981de3-2b84-45e9-a9f4-b932efcefaed kind: Event lastTimestamp: "2026-04-22T19:37:14Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-ed925-predictor-7887d85c84 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:37:14Z" name: isvc-sklearn-graph-raw-hpa-ed925-predictor.18a8c4facce87e22 namespace: kserve-ci-e2e-test resourceVersion: "15680" uid: 1cd42bf9-4310-4cca-b702-2bf143b6b0f1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:37:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-ed925-predictor namespace: kserve-ci-e2e-test resourceVersion: "15670" uid: 0d8c54e7-1932-40a5-9583-abccf40b3e1f kind: Event lastTimestamp: "2026-04-22T19:37:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:37:44Z" name: isvc-sklearn-graph-raw-hpa-ed925-predictor.18a8c4fe4bf6e806 namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: 41c2666a-980f-4649-9c42-2968ad2f2ec3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:37:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-ed925-predictor namespace: kserve-ci-e2e-test resourceVersion: "15670" uid: 0d8c54e7-1932-40a5-9583-abccf40b3e1f kind: Event lastTimestamp: "2026-04-22T19:37:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:37:44Z" name: isvc-sklearn-graph-raw-hpa-ed925-predictor.18a8c4fe4c5951b5 namespace: kserve-ci-e2e-test resourceVersion: "16025" uid: 2c5031ef-43e1-449a-8b01-badcaf1a909d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:37:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-ed925-predictor namespace: kserve-ci-e2e-test resourceVersion: "15911" uid: 0d8c54e7-1932-40a5-9583-abccf40b3e1f kind: Event lastTimestamp: "2026-04-22T19:38:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:37:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:38:59Z" name: isvc-sklearn-graph-raw-hpa-ed925-predictor.18a8c5054a98f3ae namespace: kserve-ci-e2e-test resourceVersion: "16627" uid: 8cde1d7b-e6ef-42b8-8a38-b4c55c8f1d5d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:37:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-ed925-predictor namespace: kserve-ci-e2e-test resourceVersion: "15911" uid: 0d8c54e7-1932-40a5-9583-abccf40b3e1f kind: Event lastTimestamp: "2026-04-22T19:38:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:37:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:38:59Z" name: isvc-sklearn-graph-raw-hpa-ed925-predictor.18a8c5054af81a58 namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: ad1d9f5a-fca3-4823-824c-38b038dad23a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T19:37:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-ed925 namespace: kserve-ci-e2e-test resourceVersion: "15656" uid: ee9b15d4-52bd-46a8-8971-b0b974672d5d kind: Event lastTimestamp: "2026-04-22T19:38:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:38:34Z" name: isvc-sklearn-graph-raw-hpa-ed925.18a8c4faccb2e5c3 namespace: kserve-ci-e2e-test resourceVersion: "16416" uid: face2412-a983-4a88-bd27-c56d05b77b40 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:38:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-ed925 namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: ee9b15d4-52bd-46a8-8971-b0b974672d5d kind: Event lastTimestamp: "2026-04-22T19:38:34Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-ed925] is Ready metadata: creationTimestamp: "2026-04-22T19:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:38:34Z" name: isvc-sklearn-graph-raw-hpa-ed925.18a8c50d86af79de namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: 838f11f8-f695-46e4-a8c1-bd6adb5baab3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:40:40.120178Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5 namespace: kserve-ci-e2e-test resourceVersion: "17581" uid: 456e4986-41dd-4d49-9586-74a584b69381 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5 to ip-10-0-133-159.ec2.internal metadata: creationTimestamp: "2026-04-22T19:40:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:40:40Z" name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5.18a8c52aaf6e0c2b namespace: kserve-ci-e2e-test resourceVersion: "17589" uid: c48cb149-8a5a-4004-b0b6-cfe53c6495c3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8fbbbc4c-fn4j6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:40:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5 namespace: kserve-ci-e2e-test resourceVersion: "17592" uid: 456e4986-41dd-4d49-9586-74a584b69381 kind: Event lastTimestamp: "2026-04-22T19:40:40Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:40:40Z" name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5.18a8c52ac8aeeafb namespace: kserve-ci-e2e-test resourceVersion: "17618" uid: 929ee5f2-62fd-4c41-a570-47e2f5f311ea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:40:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 456e4986-41dd-4d49-9586-74a584b69381 kind: Event lastTimestamp: "2026-04-22T19:40:40Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bcc18d6872a741f58ad083af8b7b38f37e5e040b083e0c0f5222178cff3c7b39" already present on machine metadata: creationTimestamp: "2026-04-22T19:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:40:40Z" name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5.18a8c52ac9f89607 namespace: kserve-ci-e2e-test resourceVersion: "17620" uid: 16255fa0-dd9b-4eaa-9413-3621e61cde8b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:40:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 456e4986-41dd-4d49-9586-74a584b69381 kind: Event lastTimestamp: "2026-04-22T19:40:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T19:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:40:40Z" name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5.18a8c52accb04dcf namespace: kserve-ci-e2e-test resourceVersion: "17622" uid: 8ba1f43c-21e0-42ce-b5eb-6cbd699475e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:40:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 456e4986-41dd-4d49-9586-74a584b69381 kind: Event lastTimestamp: "2026-04-22T19:40:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T19:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:40:40Z" name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5.18a8c52acd24d65f namespace: kserve-ci-e2e-test resourceVersion: "17623" uid: 8aff2a97-d9ce-4311-bfc7-ac8c73767e3e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:40:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 456e4986-41dd-4d49-9586-74a584b69381 kind: Event lastTimestamp: "2026-04-22T19:40:45Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:40:45Z" name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5.18a8c52bdc63ef33 namespace: kserve-ci-e2e-test resourceVersion: "17668" uid: c9120537-8263-4023-a947-34cdaa99e557 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:40:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 456e4986-41dd-4d49-9586-74a584b69381 kind: Event lastTimestamp: "2026-04-22T19:40:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:40:45Z" name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5.18a8c52bdfacdfe9 namespace: kserve-ci-e2e-test resourceVersion: "17672" uid: e14d111c-91e2-4640-949c-ff899af76ff7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:40:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 456e4986-41dd-4d49-9586-74a584b69381 kind: Event lastTimestamp: "2026-04-22T19:40:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:40:45Z" name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5.18a8c52be028d710 namespace: kserve-ci-e2e-test resourceVersion: "17673" uid: 209158a6-5b41-48a9-a154-b7662b483c1a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 16 eventTime: null firstTimestamp: "2026-04-22T19:40:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 456e4986-41dd-4d49-9586-74a584b69381 kind: Event lastTimestamp: "2026-04-22T19:43:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:40:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:13Z" name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5.18a8c52c1859fc4c namespace: kserve-ci-e2e-test resourceVersion: "18803" uid: 3a6400e3-3db9-4b27-b8be-fc4c2ede9d66 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 456e4986-41dd-4d49-9586-74a584b69381 kind: Event lastTimestamp: "2026-04-22T19:43:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:43:10Z" name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5.18a8c54da6da2b75 namespace: kserve-ci-e2e-test resourceVersion: "18745" uid: 67abb386-840b-4599-b66e-afbb30e2ad6b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:40:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d namespace: kserve-ci-e2e-test resourceVersion: "17561" uid: bbe06ec5-df78-42e4-ba4f-e1507b7d8a8c kind: Event lastTimestamp: "2026-04-22T19:40:40Z" message: 'Created pod: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d-flhb5' metadata: creationTimestamp: "2026-04-22T19:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:40:40Z" name: isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d.18a8c52aaefd47f7 namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: a2b527a6-c585-4555-bb76-405fb6596cf8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:40:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-0b5d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17553" uid: 51aac810-d27f-4a39-b881-2b484d49346a kind: Event lastTimestamp: "2026-04-22T19:40:40Z" message: Scaled up replica set isvc-sklearn-scale-raw-0b5d7-predictor-67945bd99d from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:40:40Z" name: isvc-sklearn-scale-raw-0b5d7-predictor.18a8c52aac206226 namespace: kserve-ci-e2e-test resourceVersion: "17564" uid: c1eaf687-de88-4862-a30b-13fc28d7ab7a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:40:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-0b5d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17559" uid: d790d3df-448a-4818-977c-5c178a5b5a60 kind: Event lastTimestamp: "2026-04-22T19:41:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:40:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:41:25Z" name: isvc-sklearn-scale-raw-0b5d7-predictor.18a8c52e2afe6929 namespace: kserve-ci-e2e-test resourceVersion: "17967" uid: e717fd03-c529-4b7b-8e56-0f21c5b250e8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:40:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-0b5d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17559" uid: d790d3df-448a-4818-977c-5c178a5b5a60 kind: Event lastTimestamp: "2026-04-22T19:41:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:40:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:41:25Z" name: isvc-sklearn-scale-raw-0b5d7-predictor.18a8c52e2b6651e0 namespace: kserve-ci-e2e-test resourceVersion: "17968" uid: 51848a4f-3e1e-42bc-8c2d-9440dddbdfb5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T19:41:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-0b5d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17743" uid: d790d3df-448a-4818-977c-5c178a5b5a60 kind: Event lastTimestamp: "2026-04-22T19:43:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:10Z" name: isvc-sklearn-scale-raw-0b5d7-predictor.18a8c538a855fa6d namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: ab0436d9-d948-428f-98ed-20822e40d4db reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T19:41:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-0b5d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17743" uid: d790d3df-448a-4818-977c-5c178a5b5a60 kind: Event lastTimestamp: "2026-04-22T19:43:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:43:10Z" name: isvc-sklearn-scale-raw-0b5d7-predictor.18a8c538a8bd4bf7 namespace: kserve-ci-e2e-test resourceVersion: "18716" uid: edc6a128-c776-435c-ba67-1f88810fa1bf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-22T19:40:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-0b5d7 namespace: kserve-ci-e2e-test resourceVersion: "17545" uid: 30e6bb1b-df71-4655-a2ca-d7fb21d17010 kind: Event lastTimestamp: "2026-04-22T19:43:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:43:03Z" name: isvc-sklearn-scale-raw-0b5d7.18a8c52aabf6faa4 namespace: kserve-ci-e2e-test resourceVersion: "18666" uid: da6b2c36-82de-4702-b648-9c257dfe4ab0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:40:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-0b5d7 namespace: kserve-ci-e2e-test resourceVersion: "17545" uid: 30e6bb1b-df71-4655-a2ca-d7fb21d17010 kind: Event lastTimestamp: "2026-04-22T19:40:40Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-0b5d7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-0b5d7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:40:40Z" name: isvc-sklearn-scale-raw-0b5d7.18a8c52aacad298e namespace: kserve-ci-e2e-test resourceVersion: "17598" uid: fb8ca660-f20c-45df-94d9-c7d76deef0b9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:40:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-0b5d7 namespace: kserve-ci-e2e-test resourceVersion: "17562" uid: 30e6bb1b-df71-4655-a2ca-d7fb21d17010 kind: Event lastTimestamp: "2026-04-22T19:40:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-0b5d7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:40:40Z" name: isvc-sklearn-scale-raw-0b5d7.18a8c52ab2646c93 namespace: kserve-ci-e2e-test resourceVersion: "17602" uid: d19fc4e4-86ea-4007-9dab-196cc72b0e9a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:43:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-0b5d7 namespace: kserve-ci-e2e-test resourceVersion: "18664" uid: 30e6bb1b-df71-4655-a2ca-d7fb21d17010 kind: Event lastTimestamp: "2026-04-22T19:43:03Z" message: InferenceService [isvc-sklearn-scale-raw-0b5d7] is Ready metadata: creationTimestamp: "2026-04-22T19:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:43:03Z" name: isvc-sklearn-scale-raw-0b5d7.18a8c54c2b0ca2fa namespace: kserve-ci-e2e-test resourceVersion: "18665" uid: 7e22d8a2-22d8-48cd-8cda-9cdf0e8b2bef reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:35:34.390728Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs namespace: kserve-ci-e2e-test resourceVersion: "14782" uid: 65e44826-b475-4bb3-b01c-33b2d1d76c7d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs to ip-10-0-133-159.ec2.internal metadata: creationTimestamp: "2026-04-22T19:35:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:35:34Z" name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs.18a8c4e38088ecf9 namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: d632aaae-6277-4dde-a298-8cea46688d52 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8fbbbc4c-fn4j6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:34Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs namespace: kserve-ci-e2e-test resourceVersion: "14796" uid: 65e44826-b475-4bb3-b01c-33b2d1d76c7d kind: Event lastTimestamp: "2026-04-22T19:35:34Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:35:34Z" name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs.18a8c4e39a034681 namespace: kserve-ci-e2e-test resourceVersion: "14810" uid: 7d958214-ed9a-4174-a504-25b99f11f88c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: 65e44826-b475-4bb3-b01c-33b2d1d76c7d kind: Event lastTimestamp: "2026-04-22T19:35:34Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bcc18d6872a741f58ad083af8b7b38f37e5e040b083e0c0f5222178cff3c7b39" already present on machine metadata: creationTimestamp: "2026-04-22T19:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:35:34Z" name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs.18a8c4e39cfd5692 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: e6db7a04-7f8d-4ad5-9e36-aac0dc0e4b3f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: 65e44826-b475-4bb3-b01c-33b2d1d76c7d kind: Event lastTimestamp: "2026-04-22T19:35:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T19:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:35:34Z" name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs.18a8c4e3a08f12d8 namespace: kserve-ci-e2e-test resourceVersion: "14817" uid: d5bfa8ee-72dc-49b1-a3ae-1aa75d2328fc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: 65e44826-b475-4bb3-b01c-33b2d1d76c7d kind: Event lastTimestamp: "2026-04-22T19:35:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T19:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:35:34Z" name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs.18a8c4e3a10e6613 namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 746bda73-3b0d-4fed-8834-67abdc7a8124 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: 65e44826-b475-4bb3-b01c-33b2d1d76c7d kind: Event lastTimestamp: "2026-04-22T19:35:39Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-22T19:35:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:35:39Z" name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs.18a8c4e49a1f73ba namespace: kserve-ci-e2e-test resourceVersion: "14864" uid: 619ce98b-727c-4e6b-a552-7866f032082b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: 65e44826-b475-4bb3-b01c-33b2d1d76c7d kind: Event lastTimestamp: "2026-04-22T19:35:58Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.5s (19.5s including waiting). Image size: 1306326779 bytes.' metadata: creationTimestamp: "2026-04-22T19:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:35:58Z" name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs.18a8c4e9246d115d namespace: kserve-ci-e2e-test resourceVersion: "15030" uid: 2e40306c-ceb9-4a2f-895f-ec456473243a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: 65e44826-b475-4bb3-b01c-33b2d1d76c7d kind: Event lastTimestamp: "2026-04-22T19:35:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:35:58Z" name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs.18a8c4e9282856cf namespace: kserve-ci-e2e-test resourceVersion: "15031" uid: 77e85c23-9103-4a28-8437-92b2dbb7a4d2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: 65e44826-b475-4bb3-b01c-33b2d1d76c7d kind: Event lastTimestamp: "2026-04-22T19:35:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:35:58Z" name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs.18a8c4e9289e19e9 namespace: kserve-ci-e2e-test resourceVersion: "15032" uid: 71816db4-36f9-47fb-b329-893dc53cfdbd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T19:35:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: 65e44826-b475-4bb3-b01c-33b2d1d76c7d kind: Event lastTimestamp: "2026-04-22T19:36:50Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:36:50Z" name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs.18a8c4e946ecac16 namespace: kserve-ci-e2e-test resourceVersion: "15417" uid: 54a8163b-7595-4c84-b5d5-31016e0d2a15 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: 65e44826-b475-4bb3-b01c-33b2d1d76c7d kind: Event lastTimestamp: "2026-04-22T19:37:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:14Z" name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs.18a8c4fad8dd47c0 namespace: kserve-ci-e2e-test resourceVersion: "15743" uid: d9cc160e-1828-4836-b97e-4fd523fa5579 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: cba0f388-b1b3-429b-a656-f97b5af70af0 kind: Event lastTimestamp: "2026-04-22T19:35:34Z" message: 'Created pod: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f-ssshs' metadata: creationTimestamp: "2026-04-22T19:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:35:34Z" name: isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f.18a8c4e37f374e03 namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 9c6453e5-b19e-4cb6-8e0d-13b3d71934c1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-8e08c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14759" uid: c84b8c86-554c-45a7-998f-80af248f3f0d kind: Event lastTimestamp: "2026-04-22T19:35:34Z" message: Scaled up replica set isvc-xgboost-graph-raw-8e08c-predictor-7b9b79c65f from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:35:34Z" name: isvc-xgboost-graph-raw-8e08c-predictor.18a8c4e378b54400 namespace: kserve-ci-e2e-test resourceVersion: "14768" uid: 5ab1faaa-b8c4-448b-93a2-bc4b86e77f0e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:35:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-8e08c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14764" uid: 28844be2-05f8-4562-95e4-4e56a4fb23aa kind: Event lastTimestamp: "2026-04-22T19:36:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:35:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:36:04Z" name: isvc-xgboost-graph-raw-8e08c-predictor.18a8c4e6f812e6a2 namespace: kserve-ci-e2e-test resourceVersion: "15078" uid: c8c8ca62-bd27-499f-acac-2c84e6737f60 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:35:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-8e08c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14764" uid: 28844be2-05f8-4562-95e4-4e56a4fb23aa kind: Event lastTimestamp: "2026-04-22T19:36:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:35:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:36:04Z" name: isvc-xgboost-graph-raw-8e08c-predictor.18a8c4e6f86e22ed namespace: kserve-ci-e2e-test resourceVersion: "15079" uid: 89f04d5e-c34b-4e99-8472-de2bdeff2b7f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:36:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-8e08c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14955" uid: 28844be2-05f8-4562-95e4-4e56a4fb23aa kind: Event lastTimestamp: "2026-04-22T19:37:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:36:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:37:04Z" name: isvc-xgboost-graph-raw-8e08c-predictor.18a8c4edf5e15847 namespace: kserve-ci-e2e-test resourceVersion: "15553" uid: bfb1a1d4-8ef5-4cbd-a5a0-f9d38984cf2c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:36:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-8e08c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14955" uid: 28844be2-05f8-4562-95e4-4e56a4fb23aa kind: Event lastTimestamp: "2026-04-22T19:37:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:36:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:37:04Z" name: isvc-xgboost-graph-raw-8e08c-predictor.18a8c4edf645e7a1 namespace: kserve-ci-e2e-test resourceVersion: "15554" uid: 47a4fca2-52bb-4dc6-9c97-c3c399c63857 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T19:35:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-8e08c namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: d9012483-5f78-4426-99dc-d3129756b6fc kind: Event lastTimestamp: "2026-04-22T19:37:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:37:00Z" name: isvc-xgboost-graph-raw-8e08c.18a8c4e37917265a namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: 067af5d1-7a51-4315-822d-0c49a44c1719 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:35:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-8e08c namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: d9012483-5f78-4426-99dc-d3129756b6fc kind: Event lastTimestamp: "2026-04-22T19:35:34Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-8e08c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-8e08c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:35:34Z" name: isvc-xgboost-graph-raw-8e08c.18a8c4e37a2a939a namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 89eb160e-9875-4f2a-b9b6-b1a6c40c82c7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-8e08c namespace: kserve-ci-e2e-test resourceVersion: "15491" uid: d9012483-5f78-4426-99dc-d3129756b6fc kind: Event lastTimestamp: "2026-04-22T19:37:00Z" message: InferenceService [isvc-xgboost-graph-raw-8e08c] is Ready metadata: creationTimestamp: "2026-04-22T19:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:37:00Z" name: isvc-xgboost-graph-raw-8e08c.18a8c4f77f3d84ec namespace: kserve-ci-e2e-test resourceVersion: "15492" uid: 21efaaa1-348c-4bf1-a9b3-83297443be0a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:37:14.596688Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp namespace: kserve-ci-e2e-test resourceVersion: "15716" uid: b831095f-5903-4cd1-9637-21c2e4549f25 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp to ip-10-0-133-159.ec2.internal metadata: creationTimestamp: "2026-04-22T19:37:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:37:14Z" name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp.18a8c4fad5467fde namespace: kserve-ci-e2e-test resourceVersion: "15723" uid: 71f8b033-2096-4080-bc13-8c981dc1019c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8fbbbc4c-fn4j6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:15Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: b831095f-5903-4cd1-9637-21c2e4549f25 kind: Event lastTimestamp: "2026-04-22T19:37:15Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:37:15Z" name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp.18a8c4faef45d94e namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: 9b961cf7-f4ad-404b-8afa-9ba593d8af6d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: b831095f-5903-4cd1-9637-21c2e4549f25 kind: Event lastTimestamp: "2026-04-22T19:37:15Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bcc18d6872a741f58ad083af8b7b38f37e5e040b083e0c0f5222178cff3c7b39" already present on machine metadata: creationTimestamp: "2026-04-22T19:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:15Z" name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp.18a8c4faf0c85b3e namespace: kserve-ci-e2e-test resourceVersion: "15764" uid: 89042983-d9a5-4614-be03-cc364611180b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: b831095f-5903-4cd1-9637-21c2e4549f25 kind: Event lastTimestamp: "2026-04-22T19:37:15Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T19:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:15Z" name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp.18a8c4faf45b37fe namespace: kserve-ci-e2e-test resourceVersion: "15768" uid: d8dcaad3-9020-409a-8fe9-5c0b23f6e785 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: b831095f-5903-4cd1-9637-21c2e4549f25 kind: Event lastTimestamp: "2026-04-22T19:37:15Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T19:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:15Z" name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp.18a8c4faf4c4ad09 namespace: kserve-ci-e2e-test resourceVersion: "15769" uid: 4012487d-519c-4e30-a92d-d91d965771e9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: b831095f-5903-4cd1-9637-21c2e4549f25 kind: Event lastTimestamp: "2026-04-22T19:37:19Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-22T19:37:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:19Z" name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp.18a8c4fbf75cec9c namespace: kserve-ci-e2e-test resourceVersion: "15817" uid: f0d9ba5a-9f9a-4188-b515-ba9929c8864c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: b831095f-5903-4cd1-9637-21c2e4549f25 kind: Event lastTimestamp: "2026-04-22T19:37:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:37:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:19Z" name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp.18a8c4fbfb5db08a namespace: kserve-ci-e2e-test resourceVersion: "15828" uid: b3aa1020-bbeb-4327-914c-4d172b8255f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: b831095f-5903-4cd1-9637-21c2e4549f25 kind: Event lastTimestamp: "2026-04-22T19:37:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:37:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:19Z" name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp.18a8c4fbfbcdf224 namespace: kserve-ci-e2e-test resourceVersion: "15829" uid: f3e3e52a-a9b8-4e00-9875-122a2df00634 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T19:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: b831095f-5903-4cd1-9637-21c2e4549f25 kind: Event lastTimestamp: "2026-04-22T19:38:11Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:38:11Z" name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp.18a8c4fc33502a91 namespace: kserve-ci-e2e-test resourceVersion: "16226" uid: 9f0ea0a5-7834-454d-9b99-2ca9b6a2f15c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: b831095f-5903-4cd1-9637-21c2e4549f25 kind: Event lastTimestamp: "2026-04-22T19:39:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:39:04Z" name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp.18a8c514847c9375 namespace: kserve-ci-e2e-test resourceVersion: "16755" uid: 9a56b24a-1c9a-4dd2-81d4-70efc51b121b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5 namespace: kserve-ci-e2e-test resourceVersion: "15707" uid: 649b3eb2-8c16-4c02-8973-d98c6c7a3863 kind: Event lastTimestamp: "2026-04-22T19:37:14Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5-sfmbp' metadata: creationTimestamp: "2026-04-22T19:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:37:14Z" name: isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5.18a8c4fad4c7cc08 namespace: kserve-ci-e2e-test resourceVersion: "15718" uid: 30a6efa8-0d89-4cd0-8915-18abb47f3f01 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-ed925-predictor namespace: kserve-ci-e2e-test resourceVersion: "15703" uid: f601a4c3-87d8-4748-aa69-642ca164c67f kind: Event lastTimestamp: "2026-04-22T19:37:14Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-ed925-predictor-55955498c5 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:37:14Z" name: isvc-xgboost-graph-raw-hpa-ed925-predictor.18a8c4fad2345b72 namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: 78f8c821-967b-4254-9887-53fa94712917 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-ed925-predictor namespace: kserve-ci-e2e-test resourceVersion: "15706" uid: 568166e4-60c2-4a04-a4fc-4390ef4d4836 kind: Event lastTimestamp: "2026-04-22T19:37:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:37:29Z" name: isvc-xgboost-graph-raw-hpa-ed925-predictor.18a8c4fe51217aff namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: eed04357-9138-4f18-8675-6380164de486 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-ed925-predictor namespace: kserve-ci-e2e-test resourceVersion: "15706" uid: 568166e4-60c2-4a04-a4fc-4390ef4d4836 kind: Event lastTimestamp: "2026-04-22T19:37:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:37:29Z" name: isvc-xgboost-graph-raw-hpa-ed925-predictor.18a8c4fe5188627b namespace: kserve-ci-e2e-test resourceVersion: "15917" uid: 9c42f1de-4d79-4d1e-baba-91069e6595ae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:37:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-ed925-predictor namespace: kserve-ci-e2e-test resourceVersion: "15916" uid: 568166e4-60c2-4a04-a4fc-4390ef4d4836 kind: Event lastTimestamp: "2026-04-22T19:38:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:38:44Z" name: isvc-xgboost-graph-raw-hpa-ed925-predictor.18a8c501d03f5b63 namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 6910419b-10b3-412e-8efd-251f4d945e48 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:37:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-ed925-predictor namespace: kserve-ci-e2e-test resourceVersion: "15916" uid: 568166e4-60c2-4a04-a4fc-4390ef4d4836 kind: Event lastTimestamp: "2026-04-22T19:38:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:38:44Z" name: isvc-xgboost-graph-raw-hpa-ed925-predictor.18a8c501d0a20947 namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 8147627b-f52a-473d-bda6-78751215cc8f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-22T19:37:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-ed925 namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: 429ffc99-cd1f-4242-8958-3599f55479f6 kind: Event lastTimestamp: "2026-04-22T19:38:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:38:21Z" name: isvc-xgboost-graph-raw-hpa-ed925.18a8c4fad2400f96 namespace: kserve-ci-e2e-test resourceVersion: "16315" uid: faffd9b2-be0b-4dc2-b9a3-394b742a1534 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-ed925 namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: 429ffc99-cd1f-4242-8958-3599f55479f6 kind: Event lastTimestamp: "2026-04-22T19:37:14Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-ed925": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-ed925": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:37:14Z" name: isvc-xgboost-graph-raw-hpa-ed925.18a8c4fad9522dbd namespace: kserve-ci-e2e-test resourceVersion: "15746" uid: 5681ba2f-8868-4296-8783-55af11923b1d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:38:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-ed925 namespace: kserve-ci-e2e-test resourceVersion: "16313" uid: 429ffc99-cd1f-4242-8958-3599f55479f6 kind: Event lastTimestamp: "2026-04-22T19:38:21Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-ed925] is Ready metadata: creationTimestamp: "2026-04-22T19:38:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:38:21Z" name: isvc-xgboost-graph-raw-hpa-ed925.18a8c50a6bc977a2 namespace: kserve-ci-e2e-test resourceVersion: "16314" uid: 957d419a-1f78-4e41-8346-8ec2183b409a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:39:04.819833Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-38710-predictor-7d9b68b89-9f6r8 namespace: kserve-ci-e2e-test resourceVersion: "16721" uid: fecab682-dbf8-473b-b88e-8227390b7c39 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-38710-predictor-7d9b68b89-9f6r8 to ip-10-0-133-159.ec2.internal metadata: creationTimestamp: "2026-04-22T19:39:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:39:04Z" name: message-dumper-raw-38710-predictor-7d9b68b89-9f6r8.18a8c5147f163c33 namespace: kserve-ci-e2e-test resourceVersion: "16729" uid: 5f0ca346-8ba6-44e9-ba73-4e9bf622e1ee reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8fbbbc4c-fn4j6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:04Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-38710-predictor-7d9b68b89-9f6r8 namespace: kserve-ci-e2e-test resourceVersion: "16734" uid: fecab682-dbf8-473b-b88e-8227390b7c39 kind: Event lastTimestamp: "2026-04-22T19:39:04Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:39:04Z" name: message-dumper-raw-38710-predictor-7d9b68b89-9f6r8.18a8c51487a3fed4 namespace: kserve-ci-e2e-test resourceVersion: "16758" uid: bae3bae0-431e-4c59-b271-cc29ea28d7ac reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-38710-predictor-7d9b68b89-9f6r8 namespace: kserve-ci-e2e-test resourceVersion: "16726" uid: fecab682-dbf8-473b-b88e-8227390b7c39 kind: Event lastTimestamp: "2026-04-22T19:39:04Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-22T19:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:39:04Z" name: message-dumper-raw-38710-predictor-7d9b68b89-9f6r8.18a8c51488cf7f3e namespace: kserve-ci-e2e-test resourceVersion: "16760" uid: c3ef361c-e635-4fa3-9aef-ddc39274284c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-38710-predictor-7d9b68b89-9f6r8 namespace: kserve-ci-e2e-test resourceVersion: "16726" uid: fecab682-dbf8-473b-b88e-8227390b7c39 kind: Event lastTimestamp: "2026-04-22T19:39:05Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.005s (1.005s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-22T19:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:39:05Z" name: message-dumper-raw-38710-predictor-7d9b68b89-9f6r8.18a8c514c4bb3b6a namespace: kserve-ci-e2e-test resourceVersion: "16767" uid: db2d3a7c-f255-4442-ae1d-7ebecc0f02e1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-38710-predictor-7d9b68b89-9f6r8 namespace: kserve-ci-e2e-test resourceVersion: "16726" uid: fecab682-dbf8-473b-b88e-8227390b7c39 kind: Event lastTimestamp: "2026-04-22T19:39:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:39:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:39:06Z" name: message-dumper-raw-38710-predictor-7d9b68b89-9f6r8.18a8c514c7ee3033 namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: 3404eecf-062a-434f-806c-8a02adfdcca4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-38710-predictor-7d9b68b89-9f6r8 namespace: kserve-ci-e2e-test resourceVersion: "16726" uid: fecab682-dbf8-473b-b88e-8227390b7c39 kind: Event lastTimestamp: "2026-04-22T19:39:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:39:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:39:06Z" name: message-dumper-raw-38710-predictor-7d9b68b89-9f6r8.18a8c514c8591d8a namespace: kserve-ci-e2e-test resourceVersion: "16769" uid: 43f8506d-799c-4d1c-b71c-262af622af65 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:40:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-38710-predictor-7d9b68b89-9f6r8 namespace: kserve-ci-e2e-test resourceVersion: "16726" uid: fecab682-dbf8-473b-b88e-8227390b7c39 kind: Event lastTimestamp: "2026-04-22T19:40:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:40:40Z" name: message-dumper-raw-38710-predictor-7d9b68b89-9f6r8.18a8c52ab498d50d namespace: kserve-ci-e2e-test resourceVersion: "17614" uid: aee78ced-ed81-4cd5-b8b7-0ff49b0f8550 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-38710-predictor-7d9b68b89 namespace: kserve-ci-e2e-test resourceVersion: "16704" uid: db443dba-530a-487e-afb7-0f542094f817 kind: Event lastTimestamp: "2026-04-22T19:39:04Z" message: 'Created pod: message-dumper-raw-38710-predictor-7d9b68b89-9f6r8' metadata: creationTimestamp: "2026-04-22T19:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:39:04Z" name: message-dumper-raw-38710-predictor-7d9b68b89.18a8c5147e9c18f3 namespace: kserve-ci-e2e-test resourceVersion: "16724" uid: 1acfeecc-8ac8-41e6-9590-6bbd09134098 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-38710-predictor namespace: kserve-ci-e2e-test resourceVersion: "16698" uid: b55d0f16-61de-46f8-8d6c-89a7d6921300 kind: Event lastTimestamp: "2026-04-22T19:39:04Z" message: Scaled up replica set message-dumper-raw-38710-predictor-7d9b68b89 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:39:04Z" name: message-dumper-raw-38710-predictor.18a8c5147b6db518 namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: c3116933-a2ea-4346-8122-04c9866d408d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-38710-predictor namespace: kserve-ci-e2e-test resourceVersion: "16703" uid: 19eef9ad-5736-424d-85f4-55d9bcd333c6 kind: Event lastTimestamp: "2026-04-22T19:39:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:39:19Z" name: message-dumper-raw-38710-predictor.18a8c517f9f35f97 namespace: kserve-ci-e2e-test resourceVersion: "16940" uid: e4120eda-8ab1-49e8-bfc4-82683e145aae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-38710-predictor namespace: kserve-ci-e2e-test resourceVersion: "16703" uid: 19eef9ad-5736-424d-85f4-55d9bcd333c6 kind: Event lastTimestamp: "2026-04-22T19:39:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:39:19Z" name: message-dumper-raw-38710-predictor.18a8c517fa5480ac namespace: kserve-ci-e2e-test resourceVersion: "16942" uid: b8c0e197-90b8-455a-870e-93f64923eb56 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-38710-predictor namespace: kserve-ci-e2e-test resourceVersion: "16941" uid: 19eef9ad-5736-424d-85f4-55d9bcd333c6 kind: Event lastTimestamp: "2026-04-22T19:39:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:39:34Z" name: message-dumper-raw-38710-predictor.18a8c51b79137435 namespace: kserve-ci-e2e-test resourceVersion: "17067" uid: 1d3df4eb-fceb-43a8-a661-9d029786896c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-38710-predictor namespace: kserve-ci-e2e-test resourceVersion: "16941" uid: 19eef9ad-5736-424d-85f4-55d9bcd333c6 kind: Event lastTimestamp: "2026-04-22T19:39:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:39:34Z" name: message-dumper-raw-38710-predictor.18a8c51b79801a1e namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: 9fdd7057-7858-4969-992c-5c3b5eb02eba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T19:39:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-38710 namespace: kserve-ci-e2e-test resourceVersion: "16691" uid: 5b9bf028-48d4-4a91-8b33-49b791910436 kind: Event lastTimestamp: "2026-04-22T19:39:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:39:06Z" name: message-dumper-raw-38710.18a8c5147b4f03bb namespace: kserve-ci-e2e-test resourceVersion: "16788" uid: 669dec98-b7db-413e-b534-b6bdace23a20 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-38710 namespace: kserve-ci-e2e-test resourceVersion: "16711" uid: 5b9bf028-48d4-4a91-8b33-49b791910436 kind: Event lastTimestamp: "2026-04-22T19:39:04Z" message: 'Failed to update status for InferenceService "message-dumper-raw-38710": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-38710": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:39:04Z" name: message-dumper-raw-38710.18a8c5147ee6e028 namespace: kserve-ci-e2e-test resourceVersion: "16728" uid: 6e377e4d-2dce-4978-87ba-a21e01382caa reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-38710 namespace: kserve-ci-e2e-test resourceVersion: "16785" uid: 5b9bf028-48d4-4a91-8b33-49b791910436 kind: Event lastTimestamp: "2026-04-22T19:39:06Z" message: InferenceService [message-dumper-raw-38710] is Ready metadata: creationTimestamp: "2026-04-22T19:39:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:39:06Z" name: message-dumper-raw-38710.18a8c514fb9f0c7a namespace: kserve-ci-e2e-test resourceVersion: "16786" uid: 3cf5e78e-0fbc-4bab-b730-5f78188892cf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:37:04.258299Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-8e08c-5f649888f5-k2t7k namespace: kserve-ci-e2e-test resourceVersion: "15536" uid: c866a62f-d697-49e0-8a01-2b9eb787aa57 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-8e08c-5f649888f5-k2t7k to ip-10-0-138-15.ec2.internal metadata: creationTimestamp: "2026-04-22T19:37:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:37:04Z" name: model-chainer-raw-8e08c-5f649888f5-k2t7k.18a8c4f86d0f3566 namespace: kserve-ci-e2e-test resourceVersion: "15543" uid: 9d88f75f-48c4-44da-ac91-9e5770d7ecfc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8fbbbc4c-fn4j6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:04Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-8e08c-5f649888f5-k2t7k namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: c866a62f-d697-49e0-8a01-2b9eb787aa57 kind: Event lastTimestamp: "2026-04-22T19:37:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-8e08c-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T19:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:04Z" name: model-chainer-raw-8e08c-5f649888f5-k2t7k.18a8c4f876f18687 namespace: kserve-ci-e2e-test resourceVersion: "15555" uid: 40820466-3da3-4e91-8e82-b91be4300ac3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-15.ec2.internal source: component: kubelet host: ip-10-0-138-15.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:05Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-8e08c-5f649888f5-k2t7k namespace: kserve-ci-e2e-test resourceVersion: "15547" uid: c866a62f-d697-49e0-8a01-2b9eb787aa57 kind: Event lastTimestamp: "2026-04-22T19:37:05Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:37:05Z" name: model-chainer-raw-8e08c-5f649888f5-k2t7k.18a8c4f8a9eedc57 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: cb5bc714-2ab5-4804-b780-b0029808f3f1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-8e08c} kind: Pod name: model-chainer-raw-8e08c-5f649888f5-k2t7k namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: c866a62f-d697-49e0-8a01-2b9eb787aa57 kind: Event lastTimestamp: "2026-04-22T19:37:05Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:3341034f2c7f09c96ac29004b065539fcf20ad6ad51aa1f097312ef91f6a2299" metadata: creationTimestamp: "2026-04-22T19:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:05Z" name: model-chainer-raw-8e08c-5f649888f5-k2t7k.18a8c4f8ab3ec443 namespace: kserve-ci-e2e-test resourceVersion: "15569" uid: a5dd9d6d-c5c7-4059-b3b8-a1bc0723c4c4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-15.ec2.internal source: component: kubelet host: ip-10-0-138-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-8e08c} kind: Pod name: model-chainer-raw-8e08c-5f649888f5-k2t7k namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: c866a62f-d697-49e0-8a01-2b9eb787aa57 kind: Event lastTimestamp: "2026-04-22T19:37:07Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:3341034f2c7f09c96ac29004b065539fcf20ad6ad51aa1f097312ef91f6a2299" in 2.406s (2.406s including waiting). Image size: 216232508 bytes.' metadata: creationTimestamp: "2026-04-22T19:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:07Z" name: model-chainer-raw-8e08c-5f649888f5-k2t7k.18a8c4f93aaf4216 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: d6748b0c-48de-4a34-a22e-ca09075fc4c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-15.ec2.internal source: component: kubelet host: ip-10-0-138-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-8e08c} kind: Pod name: model-chainer-raw-8e08c-5f649888f5-k2t7k namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: c866a62f-d697-49e0-8a01-2b9eb787aa57 kind: Event lastTimestamp: "2026-04-22T19:37:07Z" message: 'Created container: model-chainer-raw-8e08c' metadata: creationTimestamp: "2026-04-22T19:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:07Z" name: model-chainer-raw-8e08c-5f649888f5-k2t7k.18a8c4f93e0834a6 namespace: kserve-ci-e2e-test resourceVersion: "15586" uid: c0ba4cb3-3ff8-4bca-b0fe-0b1ae6525335 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-15.ec2.internal source: component: kubelet host: ip-10-0-138-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-8e08c} kind: Pod name: model-chainer-raw-8e08c-5f649888f5-k2t7k namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: c866a62f-d697-49e0-8a01-2b9eb787aa57 kind: Event lastTimestamp: "2026-04-22T19:37:07Z" message: Started container model-chainer-raw-8e08c metadata: creationTimestamp: "2026-04-22T19:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:07Z" name: model-chainer-raw-8e08c-5f649888f5-k2t7k.18a8c4f93e90ef1d namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: f2a3f2a1-7e77-4d86-a3c1-7251446285a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-15.ec2.internal source: component: kubelet host: ip-10-0-138-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-8e08c} kind: Pod name: model-chainer-raw-8e08c-5f649888f5-k2t7k namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: c866a62f-d697-49e0-8a01-2b9eb787aa57 kind: Event lastTimestamp: "2026-04-22T19:37:14Z" message: Stopping container model-chainer-raw-8e08c metadata: creationTimestamp: "2026-04-22T19:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:14Z" name: model-chainer-raw-8e08c-5f649888f5-k2t7k.18a8c4fac5b72cd1 namespace: kserve-ci-e2e-test resourceVersion: "15651" uid: 431d5d6b-c293-4f72-a554-7439148a8014 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-15.ec2.internal source: component: kubelet host: ip-10-0-138-15.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T19:37:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-8e08c} kind: Pod name: model-chainer-raw-8e08c-5f649888f5-k2t7k namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: c866a62f-d697-49e0-8a01-2b9eb787aa57 kind: Event lastTimestamp: "2026-04-22T19:37:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:37:44Z" name: model-chainer-raw-8e08c-5f649888f5-k2t7k.18a8c4fac6039234 namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: 49f44e75-f5a7-4f37-9e7c-d52c7adbd8fa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-15.ec2.internal source: component: kubelet host: ip-10-0-138-15.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-8e08c-5f649888f5 namespace: kserve-ci-e2e-test resourceVersion: "15533" uid: 793064bf-76bd-4a0b-9aba-58ce2ccaf3ce kind: Event lastTimestamp: "2026-04-22T19:37:04Z" message: 'Created pod: model-chainer-raw-8e08c-5f649888f5-k2t7k' metadata: creationTimestamp: "2026-04-22T19:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:37:04Z" name: model-chainer-raw-8e08c-5f649888f5.18a8c4f86ca703d4 namespace: kserve-ci-e2e-test resourceVersion: "15537" uid: 892178f2-1762-4ca1-bce2-827693bac828 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-8e08c namespace: kserve-ci-e2e-test resourceVersion: "15531" uid: 2d6d2955-5999-4c37-8fdc-c2a971cacc70 kind: Event lastTimestamp: "2026-04-22T19:37:04Z" message: Scaled up replica set model-chainer-raw-8e08c-5f649888f5 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:37:04Z" name: model-chainer-raw-8e08c.18a8c4f86bc74388 namespace: kserve-ci-e2e-test resourceVersion: "15534" uid: 0a489b94-6998-4430-b4c2-f50328eef439 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:37:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-8e08c namespace: kserve-ci-e2e-test resourceVersion: "15549" uid: 3ba17712-6399-4198-a17c-4feffcb1d0f0 kind: Event lastTimestamp: "2026-04-22T19:37:04Z" message: InferenceGraph [model-chainer-raw-8e08c] is Ready metadata: creationTimestamp: "2026-04-22T19:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:37:04Z" name: model-chainer-raw-8e08c.18a8c4f8704a6f1a namespace: kserve-ci-e2e-test resourceVersion: "15551" uid: 0a8e9a6e-90b1-4d1c-988c-2203ee15c0ee reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:38:54.592956Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-ed925-68b69d48f5-777g6 namespace: kserve-ci-e2e-test resourceVersion: "16571" uid: 71db560a-0382-4d72-bcad-ab0680af5d0a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-ed925-68b69d48f5-777g6 to ip-10-0-138-15.ec2.internal metadata: creationTimestamp: "2026-04-22T19:38:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:38:54Z" name: model-chainer-raw-hpa-ed925-68b69d48f5-777g6.18a8c5121d846c98 namespace: kserve-ci-e2e-test resourceVersion: "16577" uid: 416f7a7d-3c2b-44ae-9004-629d939ac110 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8fbbbc4c-fn4j6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:38:55Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-ed925-68b69d48f5-777g6 namespace: kserve-ci-e2e-test resourceVersion: "16580" uid: 71db560a-0382-4d72-bcad-ab0680af5d0a kind: Event lastTimestamp: "2026-04-22T19:38:55Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:38:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:38:55Z" name: model-chainer-raw-hpa-ed925-68b69d48f5-777g6.18a8c5123669d4fa namespace: kserve-ci-e2e-test resourceVersion: "16590" uid: 2c835ec5-03dc-4028-9fef-740cc63b32a9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:38:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-ed925} kind: Pod name: model-chainer-raw-hpa-ed925-68b69d48f5-777g6 namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: 71db560a-0382-4d72-bcad-ab0680af5d0a kind: Event lastTimestamp: "2026-04-22T19:38:55Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3341034f2c7f09c96ac29004b065539fcf20ad6ad51aa1f097312ef91f6a2299" already present on machine metadata: creationTimestamp: "2026-04-22T19:38:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:38:55Z" name: model-chainer-raw-hpa-ed925-68b69d48f5-777g6.18a8c51237a2819d namespace: kserve-ci-e2e-test resourceVersion: "16592" uid: 244b4d23-fd53-4a8b-b932-8fda982c1b36 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-15.ec2.internal source: component: kubelet host: ip-10-0-138-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:38:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-ed925} kind: Pod name: model-chainer-raw-hpa-ed925-68b69d48f5-777g6 namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: 71db560a-0382-4d72-bcad-ab0680af5d0a kind: Event lastTimestamp: "2026-04-22T19:38:55Z" message: 'Created container: model-chainer-raw-hpa-ed925' metadata: creationTimestamp: "2026-04-22T19:38:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:38:55Z" name: model-chainer-raw-hpa-ed925-68b69d48f5-777g6.18a8c5123b50f9fe namespace: kserve-ci-e2e-test resourceVersion: "16594" uid: 8a2056ac-4012-4fde-a3c4-5d86938bc9e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-15.ec2.internal source: component: kubelet host: ip-10-0-138-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:38:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-ed925} kind: Pod name: model-chainer-raw-hpa-ed925-68b69d48f5-777g6 namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: 71db560a-0382-4d72-bcad-ab0680af5d0a kind: Event lastTimestamp: "2026-04-22T19:38:55Z" message: Started container model-chainer-raw-hpa-ed925 metadata: creationTimestamp: "2026-04-22T19:38:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:38:55Z" name: model-chainer-raw-hpa-ed925-68b69d48f5-777g6.18a8c5123bc403b1 namespace: kserve-ci-e2e-test resourceVersion: "16595" uid: 158fc083-df6e-4cf6-b6d8-cd7e93413a00 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-15.ec2.internal source: component: kubelet host: ip-10-0-138-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:39:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-ed925} kind: Pod name: model-chainer-raw-hpa-ed925-68b69d48f5-777g6 namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: 71db560a-0382-4d72-bcad-ab0680af5d0a kind: Event lastTimestamp: "2026-04-22T19:39:04Z" message: Stopping container model-chainer-raw-hpa-ed925 metadata: creationTimestamp: "2026-04-22T19:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:39:04Z" name: model-chainer-raw-hpa-ed925-68b69d48f5-777g6.18a8c51474d5c34e namespace: kserve-ci-e2e-test resourceVersion: "16687" uid: bfaabd02-03cc-430c-9aee-4727f8576647 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-15.ec2.internal source: component: kubelet host: ip-10-0-138-15.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:39:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-ed925} kind: Pod name: model-chainer-raw-hpa-ed925-68b69d48f5-777g6 namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: 71db560a-0382-4d72-bcad-ab0680af5d0a kind: Event lastTimestamp: "2026-04-22T19:39:32Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:39:32Z" name: model-chainer-raw-hpa-ed925-68b69d48f5-777g6.18a8c51502f686ec namespace: kserve-ci-e2e-test resourceVersion: "17048" uid: 7f5c1c46-597a-462e-88a9-f0bf81fe1a6d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-15.ec2.internal source: component: kubelet host: ip-10-0-138-15.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:38:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-ed925-68b69d48f5 namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: a33c7ff9-bd8a-438b-9715-f72b25bca172 kind: Event lastTimestamp: "2026-04-22T19:38:54Z" message: 'Created pod: model-chainer-raw-hpa-ed925-68b69d48f5-777g6' metadata: creationTimestamp: "2026-04-22T19:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:38:54Z" name: model-chainer-raw-hpa-ed925-68b69d48f5.18a8c5121d0c5b8e namespace: kserve-ci-e2e-test resourceVersion: "16573" uid: c8879be2-7efc-402c-be52-9d310aca0dc5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:38:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-ed925 namespace: kserve-ci-e2e-test resourceVersion: "16566" uid: 2915d419-fd98-49b8-ace7-1cb6d4873819 kind: Event lastTimestamp: "2026-04-22T19:38:54Z" message: Scaled up replica set model-chainer-raw-hpa-ed925-68b69d48f5 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:38:54Z" name: model-chainer-raw-hpa-ed925.18a8c5121c079e2e namespace: kserve-ci-e2e-test resourceVersion: "16569" uid: ff6f2906-7a11-454f-afd3-83fd4ab5c4f3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:38:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-ed925 namespace: kserve-ci-e2e-test resourceVersion: "16584" uid: 4ccaa2bc-ccd5-45eb-a9d2-4404a69db10e kind: Event lastTimestamp: "2026-04-22T19:38:54Z" message: InferenceGraph [model-chainer-raw-hpa-ed925] is Ready metadata: creationTimestamp: "2026-04-22T19:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:38:54Z" name: model-chainer-raw-hpa-ed925.18a8c51220931246 namespace: kserve-ci-e2e-test resourceVersion: "16586" uid: 933d18e7-9e3b-48b6-85ef-8a7f3fe7c112 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:44:56.903096Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-942f9-predictor-8675645d6-74d87 namespace: kserve-ci-e2e-test resourceVersion: "19768" uid: 00b8dce5-b230-4852-ad0b-e23ea55db2a3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-942f9-predictor-8675645d6-74d87 to ip-10-0-133-159.ec2.internal metadata: creationTimestamp: "2026-04-22T19:44:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:44:56Z" name: raw-sklearn-942f9-predictor-8675645d6-74d87.18a8c56678e26dae namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: 569c756a-c8ec-4369-96e2-fccab9d9d624 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8fbbbc4c-fn4j6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:57Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-942f9-predictor-8675645d6-74d87 namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 00b8dce5-b230-4852-ad0b-e23ea55db2a3 kind: Event lastTimestamp: "2026-04-22T19:44:57Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:44:57Z" name: raw-sklearn-942f9-predictor-8675645d6-74d87.18a8c566926859bd namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: 868317e7-4998-4049-9370-794625163b43 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-942f9-predictor-8675645d6-74d87 namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 00b8dce5-b230-4852-ad0b-e23ea55db2a3 kind: Event lastTimestamp: "2026-04-22T19:44:57Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bcc18d6872a741f58ad083af8b7b38f37e5e040b083e0c0f5222178cff3c7b39" already present on machine metadata: creationTimestamp: "2026-04-22T19:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:57Z" name: raw-sklearn-942f9-predictor-8675645d6-74d87.18a8c56693ae8764 namespace: kserve-ci-e2e-test resourceVersion: "19790" uid: 3bad112e-e5be-420b-bb55-7a382761a0e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-942f9-predictor-8675645d6-74d87 namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 00b8dce5-b230-4852-ad0b-e23ea55db2a3 kind: Event lastTimestamp: "2026-04-22T19:44:57Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T19:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:57Z" name: raw-sklearn-942f9-predictor-8675645d6-74d87.18a8c56696ae50f0 namespace: kserve-ci-e2e-test resourceVersion: "19792" uid: e58768b0-b7c1-44cf-ac7b-8b3398e8a17a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-942f9-predictor-8675645d6-74d87 namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 00b8dce5-b230-4852-ad0b-e23ea55db2a3 kind: Event lastTimestamp: "2026-04-22T19:44:57Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T19:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:44:57Z" name: raw-sklearn-942f9-predictor-8675645d6-74d87.18a8c56697219c14 namespace: kserve-ci-e2e-test resourceVersion: "19793" uid: 4b49f1b6-c38f-4092-bad1-cfa53c159515 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:45:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-942f9-predictor-8675645d6-74d87 namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 00b8dce5-b230-4852-ad0b-e23ea55db2a3 kind: Event lastTimestamp: "2026-04-22T19:45:01Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:45:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:45:01Z" name: raw-sklearn-942f9-predictor-8675645d6-74d87.18a8c5676e41e80f namespace: kserve-ci-e2e-test resourceVersion: "19830" uid: 844f77cb-5bcb-4e71-b9fb-d2213043a4b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:45:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-942f9-predictor-8675645d6-74d87 namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 00b8dce5-b230-4852-ad0b-e23ea55db2a3 kind: Event lastTimestamp: "2026-04-22T19:45:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:45:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:45:01Z" name: raw-sklearn-942f9-predictor-8675645d6-74d87.18a8c567728848ae namespace: kserve-ci-e2e-test resourceVersion: "19834" uid: ddd3a341-0a25-416d-9d7e-299798faefea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:45:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-942f9-predictor-8675645d6-74d87 namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 00b8dce5-b230-4852-ad0b-e23ea55db2a3 kind: Event lastTimestamp: "2026-04-22T19:45:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:45:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:45:01Z" name: raw-sklearn-942f9-predictor-8675645d6-74d87.18a8c567730383d9 namespace: kserve-ci-e2e-test resourceVersion: "19836" uid: f1d1a66b-abb9-4b88-9df0-867031155d2b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-22T19:45:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-942f9-predictor-8675645d6-74d87 namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 00b8dce5-b230-4852-ad0b-e23ea55db2a3 kind: Event lastTimestamp: "2026-04-22T19:46:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:46:03Z" name: raw-sklearn-942f9-predictor-8675645d6-74d87.18a8c567aa457227 namespace: kserve-ci-e2e-test resourceVersion: "20273" uid: bb8f8875-6709-4cda-8855-e51e893f67e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:46:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-942f9-predictor-8675645d6-74d87 namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 00b8dce5-b230-4852-ad0b-e23ea55db2a3 kind: Event lastTimestamp: "2026-04-22T19:46:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:46:16Z" name: raw-sklearn-942f9-predictor-8675645d6-74d87.18a8c5791c062e3a namespace: kserve-ci-e2e-test resourceVersion: "20404" uid: e3836565-2013-4afd-bfd1-5b7ef03fcd43 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-942f9-predictor-8675645d6 namespace: kserve-ci-e2e-test resourceVersion: "19754" uid: 17666a48-dc3e-401c-a854-b0349b11c0ef kind: Event lastTimestamp: "2026-04-22T19:44:56Z" message: 'Created pod: raw-sklearn-942f9-predictor-8675645d6-74d87' metadata: creationTimestamp: "2026-04-22T19:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:56Z" name: raw-sklearn-942f9-predictor-8675645d6.18a8c5667888b5d6 namespace: kserve-ci-e2e-test resourceVersion: "19770" uid: 305daabc-057a-4053-9c08-9b61032cfd63 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-942f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 4e31dfa0-a8cd-439d-9cc0-d981fd89efc5 kind: Event lastTimestamp: "2026-04-22T19:44:56Z" message: Scaled up replica set raw-sklearn-942f9-predictor-8675645d6 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:44:56Z" name: raw-sklearn-942f9-predictor.18a8c56674e2c83d namespace: kserve-ci-e2e-test resourceVersion: "19757" uid: 0e0af34b-75c2-432e-baf0-6d5818fb21eb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:45:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-942f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19753" uid: f05009f2-abbf-4542-98dc-1c69c155bf1c kind: Event lastTimestamp: "2026-04-22T19:45:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:45:41Z" name: raw-sklearn-942f9-predictor.18a8c569f373da10 namespace: kserve-ci-e2e-test resourceVersion: "20119" uid: 318d57f5-5b79-41cb-8170-4359fab876e7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:45:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-942f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19753" uid: f05009f2-abbf-4542-98dc-1c69c155bf1c kind: Event lastTimestamp: "2026-04-22T19:45:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:45:41Z" name: raw-sklearn-942f9-predictor.18a8c569f3d5c51d namespace: kserve-ci-e2e-test resourceVersion: "20120" uid: 72af12cf-b4d2-4ad8-baf1-dab2b264a5f2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:45:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-942f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19913" uid: f05009f2-abbf-4542-98dc-1c69c155bf1c kind: Event lastTimestamp: "2026-04-22T19:46:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:45:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:46:11Z" name: raw-sklearn-942f9-predictor.18a8c574715d3231 namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: ed063d3c-e72e-4f8c-8a71-cc5efc01655b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:45:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-942f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19913" uid: f05009f2-abbf-4542-98dc-1c69c155bf1c kind: Event lastTimestamp: "2026-04-22T19:46:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:45:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:46:11Z" name: raw-sklearn-942f9-predictor.18a8c57471c0396d namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: 08a87411-1f98-4d16-8146-98ae19adae7e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-04-22T19:44:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-942f9 namespace: kserve-ci-e2e-test resourceVersion: "19745" uid: b3cddcc7-0386-4ee5-8f5e-2a3aaa7bd125 kind: Event lastTimestamp: "2026-04-22T19:46:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:46:13Z" name: raw-sklearn-942f9.18a8c56674aa85d0 namespace: kserve-ci-e2e-test resourceVersion: "20354" uid: aacfb5b3-ecc3-4920-ae95-fcaaaaebf839 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:44:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-942f9 namespace: kserve-ci-e2e-test resourceVersion: "19745" uid: b3cddcc7-0386-4ee5-8f5e-2a3aaa7bd125 kind: Event lastTimestamp: "2026-04-22T19:44:56Z" message: 'Failed to update status for InferenceService "raw-sklearn-942f9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-942f9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:56Z" name: raw-sklearn-942f9.18a8c566753d3e8f namespace: kserve-ci-e2e-test resourceVersion: "19779" uid: 9e5f300d-d515-44cf-9503-b7c2ae07e4ba reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:44:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-942f9 namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: b3cddcc7-0386-4ee5-8f5e-2a3aaa7bd125 kind: Event lastTimestamp: "2026-04-22T19:44:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-942f9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:44:56Z" name: raw-sklearn-942f9.18a8c5667a8e60c8 namespace: kserve-ci-e2e-test resourceVersion: "19780" uid: 7af3ee78-f43d-450d-a6ee-175401127132 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:46:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-942f9 namespace: kserve-ci-e2e-test resourceVersion: "20351" uid: b3cddcc7-0386-4ee5-8f5e-2a3aaa7bd125 kind: Event lastTimestamp: "2026-04-22T19:46:13Z" message: InferenceService [raw-sklearn-942f9] is Ready metadata: creationTimestamp: "2026-04-22T19:46:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:46:13Z" name: raw-sklearn-942f9.18a8c5783679f002 namespace: kserve-ci-e2e-test resourceVersion: "20352" uid: 8ce4ed5a-82e0-44e9-ac76-0b93bc6fe140 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:46:17.041768Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk namespace: kserve-ci-e2e-test resourceVersion: "20418" uid: c2f9b4ea-4a98-4a95-8f49-4d294bf355c5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk to ip-10-0-133-159.ec2.internal metadata: creationTimestamp: "2026-04-22T19:46:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:46:17Z" name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk.18a8c579218582cf namespace: kserve-ci-e2e-test resourceVersion: "20425" uid: 11e06d9d-5bba-4560-b56a-cf6db2540137 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8fbbbc4c-fn4j6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:46:17Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk namespace: kserve-ci-e2e-test resourceVersion: "20428" uid: c2f9b4ea-4a98-4a95-8f49-4d294bf355c5 kind: Event lastTimestamp: "2026-04-22T19:46:17Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:46:17Z" name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk.18a8c5793ae0e3ed namespace: kserve-ci-e2e-test resourceVersion: "20437" uid: 5ab32969-5072-49d5-8542-da2c1df4cadd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:46:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk namespace: kserve-ci-e2e-test resourceVersion: "20422" uid: c2f9b4ea-4a98-4a95-8f49-4d294bf355c5 kind: Event lastTimestamp: "2026-04-22T19:46:17Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bcc18d6872a741f58ad083af8b7b38f37e5e040b083e0c0f5222178cff3c7b39" already present on machine metadata: creationTimestamp: "2026-04-22T19:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:46:17Z" name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk.18a8c5793c214f9c namespace: kserve-ci-e2e-test resourceVersion: "20439" uid: 3e00a374-84c2-4dab-9179-e641b06cb1b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:46:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk namespace: kserve-ci-e2e-test resourceVersion: "20422" uid: c2f9b4ea-4a98-4a95-8f49-4d294bf355c5 kind: Event lastTimestamp: "2026-04-22T19:46:17Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T19:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:46:17Z" name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk.18a8c5793fa242f4 namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 985fda07-1522-4140-a94d-b006f9707bdd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:46:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk namespace: kserve-ci-e2e-test resourceVersion: "20422" uid: c2f9b4ea-4a98-4a95-8f49-4d294bf355c5 kind: Event lastTimestamp: "2026-04-22T19:46:17Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T19:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:46:17Z" name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk.18a8c579401d67b2 namespace: kserve-ci-e2e-test resourceVersion: "20442" uid: 96c670dd-2b79-4bd9-a875-05e953fa9b8b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:46:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk namespace: kserve-ci-e2e-test resourceVersion: "20422" uid: c2f9b4ea-4a98-4a95-8f49-4d294bf355c5 kind: Event lastTimestamp: "2026-04-22T19:46:22Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1256" already present on machine metadata: creationTimestamp: "2026-04-22T19:46:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:46:22Z" name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk.18a8c57a5a45f9b8 namespace: kserve-ci-e2e-test resourceVersion: "20488" uid: 598cfd75-82a2-4884-af6f-5c01451b8a17 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:46:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk namespace: kserve-ci-e2e-test resourceVersion: "20422" uid: c2f9b4ea-4a98-4a95-8f49-4d294bf355c5 kind: Event lastTimestamp: "2026-04-22T19:46:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:46:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:46:22Z" name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk.18a8c57a5d896a42 namespace: kserve-ci-e2e-test resourceVersion: "20494" uid: 8c85886e-8824-47e5-989f-fd1c5e03d874 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:46:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk namespace: kserve-ci-e2e-test resourceVersion: "20422" uid: c2f9b4ea-4a98-4a95-8f49-4d294bf355c5 kind: Event lastTimestamp: "2026-04-22T19:46:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:46:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:46:22Z" name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk.18a8c57a5e016eca namespace: kserve-ci-e2e-test resourceVersion: "20495" uid: bfd2ae48-5fb3-468b-bd91-10156edf4221 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T19:46:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk namespace: kserve-ci-e2e-test resourceVersion: "20422" uid: c2f9b4ea-4a98-4a95-8f49-4d294bf355c5 kind: Event lastTimestamp: "2026-04-22T19:47:47Z" message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:47:47Z" name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk.18a8c57a9656237f namespace: kserve-ci-e2e-test resourceVersion: "21133" uid: 4f409820-72b9-4a59-9cab-8cae0d28ef57 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk namespace: kserve-ci-e2e-test resourceVersion: "20422" uid: c2f9b4ea-4a98-4a95-8f49-4d294bf355c5 kind: Event lastTimestamp: "2026-04-22T19:47:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:47:47Z" name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk.18a8c58e2be4fd67 namespace: kserve-ci-e2e-test resourceVersion: "21129" uid: 71dd10b0-4e36-49a7-b0df-6939e1756565 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-159.ec2.internal source: component: kubelet host: ip-10-0-133-159.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:46:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4 namespace: kserve-ci-e2e-test resourceVersion: "20408" uid: eb6d93fb-0b12-46ad-b902-e24ceff8d2a7 kind: Event lastTimestamp: "2026-04-22T19:46:17Z" message: 'Created pod: raw-sklearn-runtime-1b8f0-predictor-77d78885c4-bk6xk' metadata: creationTimestamp: "2026-04-22T19:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:46:17Z" name: raw-sklearn-runtime-1b8f0-predictor-77d78885c4.18a8c5792027b8e9 namespace: kserve-ci-e2e-test resourceVersion: "20420" uid: 41e25f32-6032-449e-a63e-4d828194886b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:46:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-1b8f0-predictor namespace: kserve-ci-e2e-test resourceVersion: "20407" uid: 68acfcce-4303-4bf5-a88e-944c2a8b2939 kind: Event lastTimestamp: "2026-04-22T19:46:16Z" message: Scaled up replica set raw-sklearn-runtime-1b8f0-predictor-77d78885c4 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:46:16Z" name: raw-sklearn-runtime-1b8f0-predictor.18a8c5791d0c9f0c namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: e8595e93-24c7-4706-aeaa-f18332bf4bc5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:46:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-1b8f0-predictor namespace: kserve-ci-e2e-test resourceVersion: "20409" uid: 3dbc5fcb-5a58-42a5-b661-4fc14d8a0885 kind: Event lastTimestamp: "2026-04-22T19:46:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:46:31Z" name: raw-sklearn-runtime-1b8f0-predictor.18a8c57c9c41f744 namespace: kserve-ci-e2e-test resourceVersion: "20565" uid: dec2635e-618b-439c-b29c-b40cdf86bc96 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:46:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-1b8f0-predictor namespace: kserve-ci-e2e-test resourceVersion: "20409" uid: 3dbc5fcb-5a58-42a5-b661-4fc14d8a0885 kind: Event lastTimestamp: "2026-04-22T19:46:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:46:31Z" name: raw-sklearn-runtime-1b8f0-predictor.18a8c57c9ca871d8 namespace: kserve-ci-e2e-test resourceVersion: "20566" uid: ec938b2b-8c94-4c83-bb5d-623abc7105cf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:46:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-1b8f0-predictor namespace: kserve-ci-e2e-test resourceVersion: "20564" uid: 3dbc5fcb-5a58-42a5-b661-4fc14d8a0885 kind: Event lastTimestamp: "2026-04-22T19:47:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:47:47Z" name: raw-sklearn-runtime-1b8f0-predictor.18a8c5801b8adce6 namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: 5d5b0545-4b2d-4aa9-ab53-050ec3e18444 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:46:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-1b8f0-predictor namespace: kserve-ci-e2e-test resourceVersion: "20564" uid: 3dbc5fcb-5a58-42a5-b661-4fc14d8a0885 kind: Event lastTimestamp: "2026-04-22T19:47:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:46:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:47:47Z" name: raw-sklearn-runtime-1b8f0-predictor.18a8c5801c1ce60e namespace: kserve-ci-e2e-test resourceVersion: "21111" uid: 419545ce-9ddf-45a6-9720-fcb339338e9a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-04-22T19:46:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-1b8f0 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 89b90234-eaa3-4843-ab09-f7f9694c7808 kind: Event lastTimestamp: "2026-04-22T19:47:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:47:37Z" name: raw-sklearn-runtime-1b8f0.18a8c5791d40c83a namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: ead15ba9-f0e8-4d87-ad6f-5af8d3c0440c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:46:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-1b8f0 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 89b90234-eaa3-4843-ab09-f7f9694c7808 kind: Event lastTimestamp: "2026-04-22T19:46:17Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-1b8f0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-1b8f0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:46:17Z" name: raw-sklearn-runtime-1b8f0.18a8c5791db299f8 namespace: kserve-ci-e2e-test resourceVersion: "20430" uid: 29b345a8-1918-43e6-88f6-fe4f3aec778b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:46:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-1b8f0 namespace: kserve-ci-e2e-test resourceVersion: "20393" uid: 89b90234-eaa3-4843-ab09-f7f9694c7808 kind: Event lastTimestamp: "2026-04-22T19:46:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-1b8f0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:46:17Z" name: raw-sklearn-runtime-1b8f0.18a8c57924515245 namespace: kserve-ci-e2e-test resourceVersion: "20431" uid: c9c15aaf-51c3-434d-ad5d-a5b07b7c8f74 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:47:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-1b8f0 namespace: kserve-ci-e2e-test resourceVersion: "21041" uid: 89b90234-eaa3-4843-ab09-f7f9694c7808 kind: Event lastTimestamp: "2026-04-22T19:47:37Z" message: InferenceService [raw-sklearn-runtime-1b8f0] is Ready metadata: creationTimestamp: "2026-04-22T19:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:47:37Z" name: raw-sklearn-runtime-1b8f0.18a8c58bf391b893 namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: 6457f1f6-7dcc-4d8c-ba19-0d655a897cd9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "21327"